[Refactor] Run MHC layers on the shared boundary steps with MHC's residual operations (#41425)

This commit is contained in:
Cheng Wan
2026-09-27 16:49:42 -07:00
committed by GitHub
parent f5ae989a0e
commit 6f2d2437b6
13 changed files with 942 additions and 592 deletions
@@ -138,6 +138,52 @@ def sequence_parallel_layer_sides(
)
def scattered_residual_layer_sides(
*,
axis_sizes: Mapping[TokenAxis, int],
ffn_group: SumGroup,
is_first_layer: bool,
is_last_layer: bool,
leaves_for_reduce_scatter: bool,
) -> DecoderLayerSides:
"""A decoder layer on an input-scattered batch whose residual stays on
each attention-TP rank's slice (MHC). The attention and the FFN compute on
the full rows. The attention takes the slice and gathers it itself (its
QKV hook, or the handoff before an attention that needs the rows); the
boundary brings the FFN its rows and moves both outputs back onto the
slice: the attention output's TP sum by a reduce-scatter, the FFN's too
when it leaves it (``leaves_for_reduce_scatter``). The first layer's input
is the embedding's partial sum on the full rows; the last layer hands on
the full rows again."""
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
)
local = Layout.sharded_over(
TokenAxis.ATTN_DP,
TokenAxis.ATTN_CP,
TokenAxis.ATTN_TP_SCATTER,
axis_sizes=axis_sizes,
)
return DecoderLayerSides(
input_rows=attention if is_first_layer else local,
attention=StageInput(
attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER})
),
attention_output=StageOutput(
attention, group=SumGroup.ATTN_TP, always_leaves=True
),
ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
ffn_output=StageOutput(
attention,
group=ffn_group,
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
),
ffn_residual_rows=local,
output_rows=attention if is_last_layer else local,
input_owes=SumGroup.TP if is_first_layer else None,
)
def input_scattered_layer_sides(
*,
axis_sizes: Mapping[TokenAxis, int],
+415 -105
View File
@@ -16,7 +16,7 @@ from contextlib import contextmanager
from dataclasses import dataclass
from enum import Enum, auto
from functools import cached_property, partial
from typing import Callable, Dict, List, Optional, Tuple, Union
from typing import Callable, Dict, List, Optional, Protocol, Tuple, Union
import msgspec
import torch
@@ -46,6 +46,7 @@ from sglang.srt.layers.boundary_layout import (
TokenAxis,
decoder_layer_sides,
input_scattered_layer_sides,
scattered_residual_layer_sides,
sequence_parallel_layer_sides,
)
from sglang.srt.layers.communicator_dsa_cp import (
@@ -651,6 +652,92 @@ def _attn_input_update_and_read_residual(quant_format: str):
return _update_and_read_residual_plain
class ResidualOps(Protocol):
"""How a layer writes each stage's output into its residual and reads the
next stage's input from it. The boundary steps complete sums and move
tokens around these operations. AddAndNorm adds and normalizes; MHC's
hyper-connection streams implement the same operations (communicator_mhc)."""
# The write-back is a plain add, which one rank may run before the sum it
# adds into completes: the DP partial order, and a residual that joins the
# attention output's sum.
adds_plainly: bool
# The layer writes its FFN output into the residual itself instead of
# leaving that to the next layer's input.
updates_residual_after_ffn: bool
def enter(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""The residual the layer stack starts from, given its input."""
def read_attention_input(self, residual, norm, quant_format: str) -> Tuple:
"""The attention input and the residual, from a residual that already
holds the previous layer's output."""
def update_and_read_attention_input(
self, hidden_states, residual, norm, quant_format: str, post_residual_addition
) -> Tuple:
"""Write the previous layer's output into the residual, then read the
attention input from it."""
def update_and_read_ffn_input(self, hidden_states, residual, norm) -> Tuple:
"""Write the attention output into the residual, then read the FFN
input from it."""
def update_residual(self, hidden_states, residual) -> torch.Tensor:
"""The residual with the FFN output written into it."""
def residual_to_attn_tp_shard(self, residual, context) -> torch.Tensor:
"""This attention-TP rank's slice of the residual."""
def residual_from_attn_tp_shards(self, residual) -> torch.Tensor:
"""The residual gathered from every attention-TP rank's slice."""
class AddAndNorm:
"""A plain residual: a stage's output is added into it, and the next
stage's input is its norm. An empty batch skips the norms."""
adds_plainly = True
updates_residual_after_ffn = False
def enter(self, hidden_states):
return hidden_states
def read_attention_input(self, residual, norm, quant_format):
if residual.shape[0] == 0:
return residual, residual
return _attn_input_update_and_read_residual(quant_format)(
norm, residual, None, None
)
def update_and_read_attention_input(
self, hidden_states, residual, norm, quant_format, post_residual_addition
):
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
return _attn_input_update_and_read_residual(quant_format)(
norm, hidden_states, residual, post_residual_addition
)
def update_and_read_ffn_input(self, hidden_states, residual, norm):
if hidden_states.shape[0] == 0:
return hidden_states, residual
return norm(hidden_states, residual)
def update_residual(self, hidden_states, residual):
hidden_states += residual
return hidden_states
def residual_to_attn_tp_shard(self, residual, context):
return _redistribute_to_attn_tp_shards(residual, context)
def residual_from_attn_tp_shards(self, residual):
return _redistribute_from_attn_tp_shards(residual)
ADD_AND_NORM = AddAndNorm()
class UnreducedOutput(msgspec.Struct, frozen=True):
"""A layer output that still owes its sum, left for the next layer's input.
Hand it to the next layer, or pass it through reduce_output() before reading
@@ -730,6 +817,8 @@ class LayerCommunicator:
# Whether this class's boundary steps may be chosen from both sides'
# declarations; a subclass that picks its own steps says no.
_takes_declared_boundaries = True
# A plain residual unless the layer is built with its own.
_residual_ops: ResidualOps = ADD_AND_NORM
def __init__(
self,
@@ -744,6 +833,8 @@ class LayerCommunicator:
fused_ar_quant_keep_bf16: bool = False,
# False for a layer whose FFN always completes its own reduction.
allow_deferred_ffn_reduction: bool = True,
# How the layer writes its residual and reads its stages' inputs.
residual_ops: ResidualOps = ADD_AND_NORM,
):
self.layer_scatter_modes = layer_scatter_modes
self.input_layernorm = input_layernorm
@@ -755,6 +846,7 @@ class LayerCommunicator:
self.enable_fused_ar_quant = enable_fused_ar_quant
self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16
self.allow_deferred_ffn_reduction = allow_deferred_ffn_reduction
self._residual_ops = residual_ops
self._context = CommunicateContext.init_new()
self._context.force_layernorm_before_dp_gather = (
@@ -762,17 +854,8 @@ class LayerCommunicator:
)
# The steps the layer's ordinary batches run.
sides = self._declared_sides()
if (
sides is None
and self._takes_declared_boundaries
and (_generic_prefill_cp_shards_tokens() and _gathers_over_attention_cp())
):
# The scatter-mode steps have no attention-CP gather.
raise NotImplementedError(
"a DSA or MLA prefill CP layer outside the declarations"
)
self._steps = (
_select_boundary_steps(
self._steps_from_declarations(
sides,
fusions=self._select_mlp_input_fusions(),
force_layernorm_before_gather=force_layernorm_before_dp_gather,
@@ -783,7 +866,7 @@ class LayerCommunicator:
# The steps a batch that shards its tokens over attention CP runs; None
# without CP or where the steps come from the scatter modes.
self._cp_steps = (
_select_boundary_steps(
self._steps_from_declarations(
self._declared_sides(cp_active=True),
fusions=self._select_mlp_input_fusions(),
force_layernorm_before_gather=force_layernorm_before_dp_gather,
@@ -795,14 +878,7 @@ class LayerCommunicator:
# The steps a batch with input-scattered attention runs; None where it
# cannot run or the steps come from the scatter modes.
self._input_scattered_steps = (
_select_boundary_steps(
input_scattered_layer_sides(
axis_sizes=_token_axis_sizes(),
ffn_group=sides.ffn_output.group,
hands_on_partial=allow_reduce_scatter
and not layer_scatter_modes.is_last_layer,
)
)
self._steps_for_input_scattered(sides)
if sides is not None and self._input_can_be_scattered()
else None
)
@@ -820,7 +896,8 @@ class LayerCommunicator:
# q_lora, which input-scattered attention needs.
self._sp_steps = (
_select_boundary_steps(
sequence_parallel_layer_sides(axis_sizes=_token_axis_sizes())
sequence_parallel_layer_sides(axis_sizes=_token_axis_sizes()),
residual_ops=residual_ops,
)
if layernorm_sp.layernorm_sp_enabled()
else None
@@ -918,6 +995,7 @@ class LayerCommunicator:
attention_input, postprocess = self._post_init_communicate()
ffn_input, fused = self._select_mlp_input()
modes = self.layer_scatter_modes
ops = self._residual_ops
ffn_output = StageOutput(
self._context.layouts[modes.mlp_mode],
group=SumGroup.MOE_OUTPUT if modes.is_layer_sparse else SumGroup.TP,
@@ -940,7 +1018,7 @@ class LayerCommunicator:
ffn_output_move=(
None
if postprocess is CommunicateSummableTensorPairFn._scatter_hidden_states
else postprocess
else partial(postprocess, residual_ops=ops)
),
# Not when the way back is the MoE-CP scatter, nor when a SCATTERED
# FFN computes whole tokens with nothing left to sum.
@@ -950,9 +1028,46 @@ class LayerCommunicator:
layer_input=_complete_scattered_input,
)
def _steps_for_input_scattered(self, sides: DecoderLayerSides) -> "BoundarySteps":
"""The steps a batch with input-scattered attention runs at this layer,
for the layer's ordinary declarations ``sides``. A plain residual comes
back to the full rows inside the attention output's sum; one whose
write-back is not a plain add stays on each rank's slice."""
if self._residual_ops.adds_plainly:
scattered = input_scattered_layer_sides(
axis_sizes=_token_axis_sizes(),
ffn_group=sides.ffn_output.group,
hands_on_partial=self.allow_reduce_scatter and not self.is_last_layer,
)
handoff = _hand_qkv_hook_its_input
else:
scattered = scattered_residual_layer_sides(
axis_sizes=_token_axis_sizes(),
ffn_group=sides.ffn_output.group,
is_first_layer=self.layer_scatter_modes.is_first_layer,
is_last_layer=self.is_last_layer,
leaves_for_reduce_scatter=self.allow_reduce_scatter,
)
# DSA and hook-less attention take the slice gathered.
handoff = _hand_scattered_input_to_attention
return _select_boundary_steps(
scattered, residual_ops=self._residual_ops, attention_handoff=handoff
)
def _steps_from_declarations(
self, sides: DecoderLayerSides, **kwargs
) -> "BoundarySteps":
"""The steps this layer runs for a set of declarations."""
return _select_boundary_steps(sides, residual_ops=self._residual_ops, **kwargs)
def _post_init_communicate(self) -> Tuple[Callable, Callable]:
"""The attention input move and the postprocess the scatter modes
choose."""
if _generic_prefill_cp_shards_tokens() and _gathers_over_attention_cp():
# These tables have no attention-CP gather.
raise NotImplementedError(
"a DSA or MLA prefill CP layer outside the declarations"
)
return (
CommunicateSimpleFn.get_fn(
input_mode=self.layer_scatter_modes.layer_input_mode,
@@ -1048,7 +1163,8 @@ class LayerCommunicator:
post_residual_addition: Optional[torch.Tensor] = None,
):
self.publish_attn_lora_layout()
# The layer stack's first layer starts its residual from its input.
enters = residual is None and self.layer_scatter_modes.is_first_layer
# What the previous layer left to complete: an UnreducedOutput, or a
# producer's handoff that one of the fused entries consumes.
owed = None if isinstance(hidden_states, torch.Tensor) else hidden_states
@@ -1084,17 +1200,24 @@ class LayerCommunicator:
hidden_states, residual = layer_input(
hidden_states, residual, self._context
)
if hidden_states.shape[0] == 0:
residual = hidden_states
elif owed is not None:
ops = self._residual_ops
if enters:
hidden_states, residual = ops.enter(hidden_states), None
if owed is not None and hidden_states.shape[0] != 0:
hidden_states = reduce_output(owed)
hidden_states, residual = _attn_input_update_and_read_residual(
quant_format
)(self.input_layernorm, hidden_states, residual, post_residual_addition)
if residual is None:
# The previous layer already wrote its output into the residual.
hidden_states, residual = ops.read_attention_input(
hidden_states, self.input_layernorm, quant_format
)
else:
hidden_states, residual = _attn_input_update_and_read_residual(
quant_format
)(self.input_layernorm, hidden_states, residual, post_residual_addition)
hidden_states, residual = ops.update_and_read_attention_input(
hidden_states,
residual,
self.input_layernorm,
quant_format,
post_residual_addition,
)
return self._finish_prepare_attn(
hidden_states=hidden_states,
@@ -1122,24 +1245,27 @@ class LayerCommunicator:
def _finish_prepare_attn(self, hidden_states, residual, forward_batch):
"""Tail every prepare_attn path must run, or ``attn_inputs`` is unset."""
hidden_states = self._batch_steps(forward_batch).attention_input(
steps = self._batch_steps(forward_batch)
hidden_states = steps.attention_input(
hidden_states=hidden_states,
forward_batch=forward_batch,
context=self._context,
)
if self.qkv_latent_func is not None:
attn_inputs = AttentionInputs(
hidden_states, forward_batch, self.qkv_latent_func
)
get_attn_tp_context().set_attn_inputs(attn_inputs)
hidden_states = steps.attention_handoff(
hidden_states, forward_batch, self.qkv_latent_func
)
return hidden_states, residual
def _select_attn_input_fusions(self) -> Tuple[Callable, ...]:
"""The fused kernels that complete what the previous layer left together
with the residual update and the input norm, in the order they are tried.
Each takes (owed, residual, forward_batch, post_residual_addition) and
returns None when it does not take the batch."""
if not hasattr(self.input_layernorm, "forward_with_allreduce_fusion"):
returns None when it does not take the batch. They add the residual
plainly."""
if not (
self._residual_ops.adds_plainly
and hasattr(self.input_layernorm, "forward_with_allreduce_fusion")
):
return ()
self._attn_input_fuses_quant = (
self.enable_fused_ar_quant
@@ -1194,19 +1320,20 @@ class LayerCommunicator:
def _select_mlp_input(self) -> Tuple[Callable, Tuple["FusedMlpInput", ...]]:
"""The attention-TP -> FFN boundary's steps, chosen from the layouts, and
the fused kernels they try first. The MHC and DSA-CP communicators pick
their own implementations."""
the fused kernels they try first."""
kind = mlp_input_kind(self.layer_scatter_modes, self._context)
residual_input_mode = self.layer_scatter_modes.layer_input_mode
ops = self._residual_ops
if kind is MlpInputKind.NORM:
return _mlp_input_norm, ()
return partial(_mlp_input_norm, residual_ops=ops), ()
if kind is MlpInputKind.ATTN_TP_ALL_REDUCE:
return _mlp_input_attn_tp_all_reduce, ()
return partial(_mlp_input_attn_tp_all_reduce, residual_ops=ops), ()
if kind is MlpInputKind.SCATTER:
return (
partial(
_mlp_input_scatter,
scatters_residual=residual_input_mode == ScatterMode.TP_ATTN_FULL,
residual_ops=ops,
),
(),
)
@@ -1217,7 +1344,10 @@ class LayerCommunicator:
steps = partial(
_mlp_input_gather,
order=_mlp_input_order(
self._context, residual_input_mode, tuple(f.run for f in fusions)
self._context,
residual_input_mode,
tuple(f.run for f in fusions),
residual_ops=ops,
),
)
if kind is MlpInputKind.GATHER_MOE_CP:
@@ -1226,8 +1356,11 @@ class LayerCommunicator:
def _select_mlp_input_fusions(self) -> Tuple["FusedMlpInput", ...]:
"""The fused kernels that can take the attention -> FFN steps, in the
order they are tried."""
if not hasattr(self.post_attention_layernorm, "forward_with_allreduce_fusion"):
order they are tried. They add the residual plainly."""
if not (
self._residual_ops.adds_plainly
and hasattr(self.post_attention_layernorm, "forward_with_allreduce_fusion")
):
return ()
return (
FusedMlpInput(
@@ -1286,25 +1419,23 @@ class LayerCommunicator:
residual: torch.Tensor,
forward_batch: ForwardBatch,
):
steps = self._batch_steps(forward_batch)
if steps.returns_over_dp:
step = self._postprocess_dp_step(forward_batch) or _redistribute_output
return _to_local_tokens(step, forward_batch, hidden_states), residual
return steps.ffn_output_move(
hidden_states=hidden_states,
residual=residual,
return self._complete_ffn_output_now(
hidden_states,
residual,
forward_batch=forward_batch,
context=self._context,
allow_reduce_scatter=self.allow_reduce_scatter,
is_layer_sparse=self.layer_scatter_modes.is_layer_sparse,
dp_step=self._postprocess_dp_step(forward_batch),
)
def _local_token_move_can_go_to_next_layer(
self, forward_batch: ForwardBatch
) -> bool:
"""Whether the next layer's input can run this layer's move of its FFN
output back to this rank's tokens: the base postprocess scatter."""
return self._batch_steps(forward_batch).returns_over_dp
output back to this rank's tokens: the base postprocess scatter, when
the next layer's input also writes the output into the residual."""
return (
self._batch_steps(forward_batch).returns_over_dp
and not self._residual_ops.updates_residual_after_ffn
)
def _postprocess_dp_step(
self, forward_batch: ForwardBatch
@@ -1418,16 +1549,27 @@ class LayerCommunicator:
forward_batch: ForwardBatch,
dp_step: Optional[Callable],
) -> Tuple[torch.Tensor, torch.Tensor]:
"""This layer's postprocess, run with the attention-DP step the FFN exit
chose."""
if self._local_token_move_can_go_to_next_layer(forward_batch):
return (
_to_local_tokens(
dp_step or _redistribute_output, forward_batch, hidden_states
),
residual,
"""This layer's postprocess, run with the attention-DP step already
chosen: the move back to where the next layer reads the FFN output, then
the write-back into the residual for a layer that does it itself."""
steps = self._batch_steps(forward_batch)
if steps.returns_over_dp:
hidden_states = _to_local_tokens(
dp_step or _redistribute_output, forward_batch, hidden_states
)
return self.postprocess_layer(hidden_states, residual, forward_batch)
else:
hidden_states, residual = steps.ffn_output_move(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
context=self._context,
allow_reduce_scatter=self.allow_reduce_scatter,
is_layer_sparse=self.layer_scatter_modes.is_layer_sparse,
)
if residual is not None and self._residual_ops.updates_residual_after_ffn:
hidden_states = self._residual_ops.update_residual(hidden_states, residual)
residual = None
return hidden_states, residual
def ffn_exit(self, forward_batch: ForwardBatch) -> "FfnExit":
"""Decide once how this layer's FFN output reduction completes. Use the
@@ -1751,7 +1893,9 @@ class CommunicateSimpleFn:
def _redistribute_from_attn_tp_shards(tensor: torch.Tensor) -> torch.Tensor:
gathered = get_local_dp_buffer(get_parallel().attn_tp_group)
gathered = get_local_dp_buffer(
get_parallel().attn_tp_group, hidden_size=tensor.shape[-1]
)
attn_tp_all_gather_into_tensor(gathered, tensor)
return gathered
@@ -1808,10 +1952,11 @@ def _redistribute_input_to_moe_cp(
def _mlp_input_reduce_output(
hidden_states: torch.Tensor, forward_batch: ForwardBatch
hidden_states: torch.Tensor, forward_batch: ForwardBatch, may_quantize: bool = True
) -> torch.Tensor:
if (
not forward_batch.forward_mode.is_decode_or_idle()
may_quantize
and not forward_batch.forward_mode.is_decode_or_idle()
and get_exec().comm.enable_quant_communications
):
return attention_tensor_model_parallel_quant_all_reduce(hidden_states)
@@ -1873,17 +2018,21 @@ def _mlp_input_without_dp(
*,
gathers_residual: bool,
fusions: Tuple[Callable, ...],
residual_ops: ResidualOps = ADD_AND_NORM,
):
if gathers_residual:
residual = _redistribute_from_attn_tp_shards(residual)
residual = residual_ops.residual_from_attn_tp_shards(residual)
for fused in fusions:
result = fused(hidden_states, residual, forward_batch)
if result is not None:
return result
hidden_states = _mlp_input_reduce_output(hidden_states, forward_batch)
# MHC sums its streams in full precision.
hidden_states = _mlp_input_reduce_output(
hidden_states, forward_batch, may_quantize=residual_ops.adds_plainly
)
if _is_npu and context.cache is not None:
_ = prepare_weight_cache(hidden_states, context.cache)
return layernorm(hidden_states, residual)
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
def _mlp_input_dp_replicate(
@@ -1896,12 +2045,14 @@ def _mlp_input_dp_replicate(
gathers_residual: bool,
reduces_attention_tp: bool,
places_cp_shards: bool = False,
residual_ops: ResidualOps = ADD_AND_NORM,
):
"""Attention DP: complete the attention-TP sum if it is owed, add and
normalize locally, then gather. With ``places_cp_shards`` each CP rank puts
its shard of the DP group's tokens beside the others' in the group's slot."""
"""Attention DP: complete the attention-TP sum if it is owed, write the
output into the residual and read the FFN input locally, then gather. With
``places_cp_shards`` each CP rank puts its shard of the DP group's tokens
beside the others' in the group's slot."""
if gathers_residual:
residual = _redistribute_from_attn_tp_shards(residual)
residual = residual_ops.residual_from_attn_tp_shards(residual)
if hidden_states.shape[0] != 0:
if reduces_attention_tp:
hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states)
@@ -1909,7 +2060,13 @@ def _mlp_input_dp_replicate(
get_parallel().tp_group,
disabled=not is_allocation_symmetric(),
):
hidden_states, residual = layernorm(hidden_states, residual)
hidden_states, residual = residual_ops.update_and_read_ffn_input(
hidden_states, residual, layernorm
)
else:
hidden_states, residual = residual_ops.update_and_read_ffn_input(
hidden_states, residual, layernorm
)
cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None
return (
_redistribute_input_to_dp(hidden_states, forward_batch, cp_shard_counts),
@@ -1948,6 +2105,7 @@ def _mlp_input_order(
context: CommunicateContext,
residual_input_mode: ScatterMode,
fusions: Tuple[Callable, ...],
residual_ops: ResidualOps = ADD_AND_NORM,
) -> Callable:
"""The steps from the attention output to the FFN input, chosen from facts
fixed at construction."""
@@ -1956,13 +2114,21 @@ def _mlp_input_order(
)
if context.attn_dp_size == 1:
return partial(
_mlp_input_without_dp, gathers_residual=gathers_residual, fusions=fusions
_mlp_input_without_dp,
gathers_residual=gathers_residual,
fusions=fusions,
residual_ops=residual_ops,
)
if context.force_layernorm_before_dp_gather or context.attn_tp_size == 1:
if (
context.force_layernorm_before_dp_gather
or context.attn_tp_size == 1
or not residual_ops.adds_plainly
):
return partial(
_mlp_input_dp_replicate,
gathers_residual=gathers_residual,
reduces_attention_tp=context.attn_tp_size > 1,
residual_ops=residual_ops,
)
return partial(_mlp_input_dp_partial, gathers_residual=gathers_residual)
@@ -2049,6 +2215,53 @@ def _same_ranks(a: GroupCoordinator, b: GroupCoordinator) -> bool:
return sorted(a.ranks) == sorted(b.ranks)
def _hand_qkv_hook_its_input(
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
qkv_latent_func: Optional[Callable],
) -> torch.Tensor:
"""Give the attention's QKV hook the attention input."""
if qkv_latent_func is not None:
get_attn_tp_context().set_attn_inputs(
AttentionInputs(hidden_states, forward_batch, qkv_latent_func)
)
return hidden_states
def _hand_scattered_input_to_attention(
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
qkv_latent_func: Optional[Callable],
) -> torch.Tensor:
"""Input-scattered attention takes each rank's slice, and its QKV hook
gathers the rows after the projection. DSA and attention without a hook
consume full hidden states, so those are gathered here, and the hook is
told they are."""
ctx = get_attn_tp_context()
if ctx.is_dsa or qkv_latent_func is None:
hidden_states = _tp_all_gather_scattered_rows(hidden_states, forward_batch)
if qkv_latent_func is not None:
ctx.set_attn_inputs(
AttentionInputs(
hidden_states,
forward_batch,
qkv_latent_func,
is_pre_gathered=ctx.is_dsa,
)
)
return hidden_states
def _tp_all_gather_scattered_rows(
hidden_states: torch.Tensor, forward_batch: ForwardBatch
) -> torch.Tensor:
# Input-scattered attention keeps the same number of tokens on every TP rank.
total_tokens = forward_batch.input_ids.shape[0]
output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1]))
get_parallel().tp_group.all_gather_into_tensor(output, hidden_states)
return output
class BoundarySteps(msgspec.Struct, frozen=True):
"""The steps a batch runs at a layer's boundaries: into the attention,
from the attention output to the FFN input, and the FFN output on to the
@@ -2070,6 +2283,9 @@ class BoundarySteps(msgspec.Struct, frozen=True):
# Completes what the layer's input owes before the input norm; None when it
# owes nothing.
layer_input: Optional[Callable] = None
# Hands the attention its input once attention_input has moved it:
# (hidden_states, forward_batch, qkv_latent_func) -> hidden_states.
attention_handoff: Callable = _hand_qkv_hook_its_input
@property
def returns_over_dp(self) -> bool:
@@ -2082,14 +2298,17 @@ def _select_boundary_steps(
fusions: Tuple["FusedMlpInput", ...] = (),
force_layernorm_before_gather: bool = False,
cp_moves: Optional[CpMoves] = None,
residual_ops: ResidualOps = ADD_AND_NORM,
attention_handoff: Callable = _hand_qkv_hook_its_input,
) -> BoundarySteps:
"""The steps a set of declarations chooses; ``cp_moves`` for the ones
that gather over attention CP."""
"""The steps a set of declarations chooses, around the layer's residual
operations; ``cp_moves`` for the ones that gather over attention CP."""
returns_over_dp, ffn_output_move, completes_sum = _select_ffn_output_move(
sides.ffn_output,
residual=sides.ffn_residual_rows,
to=sides.output_rows,
cp_moves=cp_moves,
residual_ops=residual_ops,
)
rows = sides.input_rows
layer_input = None
@@ -2113,6 +2332,7 @@ def _select_boundary_steps(
fusions=fusions,
residual_joins_sum=sides.residual_joins_attention_sum,
cp_moves=cp_moves,
residual_ops=residual_ops,
)
return BoundarySteps(
attention_input=_select_attention_input_move(rows, sides.attention),
@@ -2123,6 +2343,7 @@ def _select_boundary_steps(
ffn_sum_is_movable=sides.ffn_output.group is not None,
fused=fused,
layer_input=layer_input,
attention_handoff=attention_handoff,
)
@@ -2164,13 +2385,16 @@ def _select_ffn_input(
fusions: Tuple[FusedMlpInput, ...],
residual_joins_sum: bool = False,
cp_moves: Optional[CpMoves] = None,
residual_ops: ResidualOps = ADD_AND_NORM,
) -> Tuple[Callable, Tuple[FusedMlpInput, ...]]:
"""The steps from the attention output to the FFN input, and the fused
kernels they try first: complete the attention-TP sum, move the residual to
the rows it has while the FFN runs, add it and normalize, and bring the rows
to what the FFN's group needs: a gather over attention DP, or each rank's
own slice. A kernel in ``fusions`` is tried only when nothing is gathered or
sliced, and only if it completes the sum the attention output owes."""
the rows it has while the FFN runs, write the output into it and read the
FFN input, and bring the rows to what the FFN's group needs: a gather over
attention DP, or each rank's own slice. A kernel in ``fusions`` is tried
only when nothing is gathered or sliced, and only if it completes the sum
the attention output owes. A write-back that is not a plain add runs only
after the sum completes."""
# What the attention output owes decides the steps: the attention-TP sum,
# always left by the output projection, or nothing.
owes_attention_tp = produced.group is SumGroup.ATTN_TP
@@ -2193,9 +2417,19 @@ def _select_ffn_input(
):
raise NotImplementedError(f"{produced=} {residual=} {need=}")
return (
partial(_mlp_input_scatter, scatters_residual=residual != residual_to),
partial(
_mlp_input_scatter,
scatters_residual=residual != residual_to,
residual_ops=residual_ops,
),
(),
)
if residual_to.sharded - produced.layout.sharded == {TokenAxis.ATTN_TP_SCATTER}:
# The residual stays on each rank's slice while the FFN takes the
# attention's rows (MHC on an input-scattered batch).
if gathered or not owes_attention_tp or residual != residual_to:
raise NotImplementedError(f"{produced=} {residual=} {need=}")
return partial(_mlp_input_on_residual_shard, residual_ops=residual_ops), ()
if gathered == {TokenAxis.ATTN_CP}:
# Each CP rank completes its own chunk, then the CP moves gather them.
if cp_moves is None:
@@ -2208,6 +2442,7 @@ def _select_ffn_input(
force_layernorm_before_gather=force_layernorm_before_gather,
fusions=fusions,
residual_joins_sum=residual_joins_sum,
residual_ops=residual_ops,
)
return partial(cp_moves.gather, gather=on_chunk), fused
if (
@@ -2226,10 +2461,12 @@ def _select_ffn_input(
gathers_residual = residual != residual_to
if not gathered:
if not owes_attention_tp:
return _mlp_input_norm, ()
return partial(_mlp_input_norm, residual_ops=residual_ops), ()
if gathers_residual and residual_joins_sum:
# Each rank adds its slice of the residual into its share of the
# sum, so the all-reduce also brings the residual back to every row.
if not residual_ops.adds_plainly:
raise NotImplementedError(f"{produced=} {residual=} {need=}")
return _mlp_input_residual_into_sum, ()
fused = tuple(f for f in fusions if f.completes is produced.group)
return (
@@ -2237,6 +2474,7 @@ def _select_ffn_input(
_mlp_input_without_dp,
gathers_residual=gathers_residual,
fusions=tuple(f.run for f in fused),
residual_ops=residual_ops,
),
fused,
)
@@ -2246,7 +2484,11 @@ def _select_ffn_input(
# Over attention DP and CP, the DP gather puts each CP rank's shard in its
# DP group's slot, so the one DP sum gathers both axes.
places_cp_shards = TokenAxis.ATTN_CP in gathered
if owes_attention_tp and not force_layernorm_before_gather:
if (
owes_attention_tp
and not force_layernorm_before_gather
and residual_ops.adds_plainly
):
return (
partial(
_mlp_input_dp_partial,
@@ -2261,6 +2503,7 @@ def _select_ffn_input(
gathers_residual=gathers_residual,
reduces_attention_tp=owes_attention_tp,
places_cp_shards=places_cp_shards,
residual_ops=residual_ops,
),
(),
)
@@ -2272,21 +2515,38 @@ def _select_ffn_output_move(
residual: Layout,
to: Layout,
cp_moves: Optional[CpMoves] = None,
residual_ops: ResidualOps = ADD_AND_NORM,
) -> Tuple[bool, Optional[Callable], bool]:
"""How the FFN output reaches the rows the layer hands on: whether it goes
back by undoing the attention-DP gather (the FFN exit and postprocess run
that step), or else the postprocess that moves it, None when there is none
to choose here; and whether that move also completes the sum the FFN
leaves."""
pair = CommunicateSummableTensorPairFn
if produced.layout == residual:
if to == residual:
return False, CommunicateSummableTensorPairFn._trivial, False
return False, pair._trivial, False
if to.sharded == residual.sharded - {TokenAxis.ATTN_TP_SCATTER}:
# Each rank's slice back to the attention's rows: fold the residual
# into the output, then gather over attention TP.
return False, CommunicateSummableTensorPairFn._gather, False
# Each rank's slice back to the attention's rows: write the output
# into the residual, then gather over attention TP.
return False, partial(pair._gather, residual_ops=residual_ops), False
raise NotImplementedError(f"{produced=} {residual=} {to=}")
returned = residual.sharded - produced.layout.sharded
if returned == {TokenAxis.ATTN_TP_SCATTER} and to in (residual, produced.layout):
# The residual stays on each rank's slice (MHC on an input-scattered
# batch): a reduce-scatter onto the slice completes the sum the FFN
# leaves; a complete output is only sliced.
sums = produced.leaves_for_reduce_scatter
return (
False,
partial(
pair._onto_residual_shard,
sums=sums,
gathers_back=to != residual,
residual_ops=residual_ops,
),
sums,
)
if to != residual or not produced.layout.sharded <= residual.sharded:
raise NotImplementedError(f"{produced=} {residual=} {to=}")
if returned == {TokenAxis.ATTN_CP}:
@@ -2371,11 +2631,10 @@ def _mlp_input_norm(
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
residual_ops: ResidualOps = ADD_AND_NORM,
):
# TODO move these `if shape != 0` into LayerNorm itself
if hidden_states.shape[0] != 0:
hidden_states, residual = layernorm(hidden_states, residual)
return hidden_states, residual
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
def _mlp_input_attn_tp_all_reduce(
@@ -2384,6 +2643,8 @@ def _mlp_input_attn_tp_all_reduce(
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
residual_ops: ResidualOps = ADD_AND_NORM,
):
"""All-reduce hidden states inside the attention TP group, then layernorm.
@@ -2392,9 +2653,7 @@ def _mlp_input_attn_tp_all_reduce(
``TP_ATTN_FULL`` across the boundary.
"""
hidden_states = get_parallel().attn_tp_group.all_reduce(hidden_states)
if hidden_states.shape[0] != 0:
hidden_states, residual = layernorm(hidden_states, residual)
return hidden_states, residual
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
def _mlp_input_scatter(
@@ -2405,14 +2664,35 @@ def _mlp_input_scatter(
context: CommunicateContext,
*,
scatters_residual: bool,
residual_ops: ResidualOps = ADD_AND_NORM,
):
hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards(
hidden_states, context
)
if scatters_residual:
residual = _redistribute_to_attn_tp_shards(residual, context)
if hidden_states.shape[0] != 0:
hidden_states, residual = layernorm(hidden_states, residual)
residual = residual_ops.residual_to_attn_tp_shard(residual, context)
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
def _mlp_input_on_residual_shard(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
residual_ops: ResidualOps,
):
"""The residual stays on each rank's slice while the FFN takes the full
rows: reduce-scatter the attention output onto the slice, which completes
its sum, write it into the residual and read the FFN input there, then
gather the input back into the attention output's rows."""
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
shard = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
get_parallel().tp_group.reduce_scatter_tensor(shard, hidden_states)
shard, residual = residual_ops.update_and_read_ffn_input(shard, residual, layernorm)
attn_tp_all_gather_into_tensor(hidden_states, shard)
return hidden_states, residual
@@ -2681,11 +2961,41 @@ class CommunicateSummableTensorPairFn:
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
residual_ops: ResidualOps = ADD_AND_NORM,
**kwargs,
):
hidden_states += residual
hidden_states = residual_ops.update_residual(hidden_states, residual)
return _redistribute_from_attn_tp_shards(hidden_states), None
@staticmethod
def _onto_residual_shard(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
*,
sums: bool,
gathers_back: bool,
residual_ops: ResidualOps,
**kwargs,
):
"""Bring the FFN output onto the slice of the rows the residual is on:
a reduce-scatter that also completes its sum when ``sums``, else this
rank's slice of the complete output. With ``gathers_back`` it is
written into the residual there and the full rows are gathered back."""
if sums:
hidden_states, _ = tp_reduce_scatter(hidden_states, None, context)
else:
hidden_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
if not gathers_back:
return hidden_states, residual
local_states = residual_ops.update_residual(hidden_states, residual)
hidden_states = local_states.new_empty(
local_states.shape[0] * context.tp_size, *local_states.shape[1:]
)
get_parallel().tp_group.all_gather_into_tensor(hidden_states, local_states)
return hidden_states, None
@staticmethod
def _scatter(
hidden_states: torch.Tensor,
+69 -417
View File
@@ -12,62 +12,28 @@
# limitations under the License.
# ==============================================================================
from dataclasses import dataclass
from functools import partial
from typing import Callable, Optional
import torch
from sglang.kernels.ops.layernorm.mhc import hc_contract, hc_expand
from sglang.srt.distributed.communication_op import (
attention_tensor_model_parallel_all_reduce,
)
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
use_symmetric_memory,
)
from sglang.srt.layers.boundary_layout import DecoderLayerSides, TokenAxis
from sglang.srt.layers.communicator import (
AttentionInputs,
CommunicateContext,
CommunicateSimpleFn,
CommunicateSummableTensorPairFn,
FfnCompletion,
BoundarySteps,
LayerCommunicator,
LayerScatterModes,
MlpInputKind,
ScatterMode,
get_attn_tp_context,
mlp_input_kind,
tp_reduce_scatter,
)
from sglang.srt.layers.dp_attention import (
attn_tp_all_gather_into_tensor,
attn_tp_reduce_scatter_tensor,
dp_gather_replicate,
dp_reduce_scatter_tensor,
dp_scatter,
get_dp_global_num_tokens,
get_global_dp_buffer,
get_local_dp_buffer_mhc,
is_allocation_symmetric,
)
from sglang.srt.layers.moe import should_use_dp_reduce_scatterv
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import get_parallel
def tp_all_gather_hidden_states(hidden_states, forward_batch):
assert get_attn_tp_context().input_scattered, (
"Input scattered guarantees same num tokens in TP group."
)
total_tokens = forward_batch.input_ids.shape[0]
output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1]))
get_parallel().tp_group.all_gather_into_tensor(output, hidden_states)
return output
@dataclass
class MHCState:
"""Parameters belong to the owning layer; this state only holds scratch
"""A layer's residual as hyper-connection streams, with the residual
operations the boundary steps run (communicator.ResidualOps): the residual
is hc_mult streams, a stage's output is written in with hc_post and the
next stage's input read with hc_pre and the norm. A read produces the
h_res / h_post the next write-back consumes; they move with the tokens.
Parameters belong to the owning layer; this state only holds scratch
shared across communication stages."""
hc_mult: int
@@ -75,9 +41,19 @@ class MHCState:
hc_ffn_pre: Callable
hc_post: Callable
hc_ffn_post_pre: Optional[Callable] = None
# The last layer's write-back also contracts the streams into the hidden
# states the layer stack hands on.
is_last_layer: bool = False
h_res: Optional[torch.Tensor] = None
h_post: Optional[torch.Tensor] = None
# hc_post is not a plain add, so it runs only once the sum it writes in
# is complete.
adds_plainly = False
# The FFN output's write-back uses the coefficients this layer's FFN input
# read produced, so the layer runs it itself.
updates_residual_after_ffn = True
@staticmethod
def _resolve_out_norm(out_norm):
if out_norm is None:
@@ -131,261 +107,45 @@ class MHCState:
self.h_res = None
self.h_post = None
def enter(self, hidden_states):
return hc_expand(hidden_states, self.hc_mult)
class MHCCommunicateWithAllReduceAndLayerNormFn:
@staticmethod
def _scatter_hidden_states_and_residual(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
residual_input_mode,
mhc: MHCState,
def read_attention_input(self, residual, norm, quant_format):
if quant_format:
raise NotImplementedError(f"an MHC attention input in {quant_format=}")
return self.attn_split(residual, out_norm=norm)
def update_and_read_attention_input(
self, hidden_states, residual, norm, quant_format, post_residual_addition
):
input_hidden_states = hidden_states
hidden_states = hidden_states.tensor_split(context.attn_tp_size)[
context.attn_tp_rank
]
attn_tp_reduce_scatter_tensor(hidden_states, input_hidden_states)
if residual_input_mode == ScatterMode.TP_ATTN_FULL:
residual = residual.tensor_split(context.attn_tp_size)[context.attn_tp_rank]
mhc.h_res = mhc.h_res.tensor_split(context.attn_tp_size)[
context.attn_tp_rank
]
mhc.h_post = mhc.h_post.tensor_split(context.attn_tp_size)[
context.attn_tp_rank
]
# An MHC layer's input arrives with the previous FFN output written in.
raise NotImplementedError("an MHC layer takes its input written back")
hidden_states, residual = mhc.attn_to_mlp(
hidden_states, residual, out_norm=layernorm
def update_and_read_ffn_input(self, hidden_states, residual, norm):
return self.attn_to_mlp(hidden_states, residual, out_norm=norm)
def update_residual(self, hidden_states, residual):
hidden_states = self.mlp_combine(hidden_states, residual)
self.reset_aux()
if self.is_last_layer:
hidden_states = hc_contract(hidden_states, self.hc_mult)
return hidden_states
def residual_to_attn_tp_shard(self, residual, context):
rank, size = context.attn_tp_rank, context.attn_tp_size
self.h_res = self.h_res.tensor_split(size)[rank]
self.h_post = self.h_post.tensor_split(size)[rank]
return residual.tensor_split(size)[rank]
def residual_from_attn_tp_shards(self, residual):
raise NotImplementedError(
"Unsupported: h_res/h_post allgather not implemented."
)
return hidden_states, residual
@staticmethod
def _simple(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
mhc: MHCState,
):
hidden_states, residual = mhc.attn_to_mlp(
hidden_states, residual, out_norm=layernorm
)
return hidden_states, residual
@staticmethod
def _tp_all_reduce_with_scattered_residual(
hidden_states: torch.Tensor,
residual: torch.Tensor,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
mhc: MHCState,
):
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
scatter_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
get_parallel().tp_group.reduce_scatter_tensor(scatter_states, hidden_states)
scatter_states, residual = mhc.attn_to_mlp(
scatter_states, residual, out_norm=layernorm
)
attn_tp_all_gather_into_tensor(hidden_states, scatter_states)
return hidden_states, residual
@staticmethod
def _gather_hidden_states_and_residual(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
residual_input_mode,
mhc: MHCState,
):
if get_attn_tp_context().input_scattered:
return MHCCommunicateWithAllReduceAndLayerNormFn._tp_all_reduce_with_scattered_residual(
hidden_states,
residual,
layernorm,
context,
mhc=mhc,
)
if residual_input_mode == ScatterMode.SCATTERED and context.attn_tp_size > 1:
raise NotImplementedError(
"Unsupported: h_res/h_post allgather not implemented."
)
hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states)
if context.attn_dp_size != 1:
if hidden_states.shape[0] != 0:
with use_symmetric_memory(
get_parallel().tp_group,
disabled=not is_allocation_symmetric(),
):
hidden_states, residual = mhc.attn_to_mlp(
hidden_states, residual, out_norm=layernorm
)
else:
hidden_states, residual = mhc.attn_to_mlp(hidden_states, residual)
hidden_states, local_hidden_states = (
get_global_dp_buffer(get_parallel().tp_group),
hidden_states,
)
dp_gather_replicate(hidden_states, local_hidden_states, forward_batch)
else:
hidden_states, residual = mhc.attn_to_mlp(
hidden_states, residual, out_norm=layernorm
)
return hidden_states, residual
class MHCCommunicateSummableTensorPairFn(CommunicateSummableTensorPairFn):
@staticmethod
def get_fn(
hidden_states_input_mode: ScatterMode,
residual_input_mode: ScatterMode,
output_mode: ScatterMode,
context: CommunicateContext,
):
fn = CommunicateSummableTensorPairFn.get_fn(
hidden_states_input_mode,
residual_input_mode,
output_mode,
context,
)
replacements = {
CommunicateSummableTensorPairFn._trivial: MHCCommunicateSummableTensorPairFn._trivial,
CommunicateSummableTensorPairFn._scatter_hidden_states: MHCCommunicateSummableTensorPairFn._scatter_hidden_states,
CommunicateSummableTensorPairFn._gather: MHCCommunicateSummableTensorPairFn._gather,
CommunicateSummableTensorPairFn._scatter: MHCCommunicateSummableTensorPairFn._scatter,
}
return replacements.get(fn, fn)
@staticmethod
def _trivial(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
*,
mhc: MHCState,
is_last_layer: bool,
**kwargs,
):
if get_attn_tp_context().input_scattered:
hidden_states, _ = tp_reduce_scatter(hidden_states, None, context)
hidden_states = mhc.mlp_combine(hidden_states, residual)
if not is_last_layer:
return hidden_states, None
hidden_states = hc_contract(hidden_states, mhc.hc_mult)
if get_attn_tp_context().input_scattered:
local_states = hidden_states
hidden_states = local_states.new_empty(
local_states.shape[0] * context.tp_size, *local_states.shape[1:]
)
get_parallel().tp_group.all_gather_into_tensor(hidden_states, local_states)
return hidden_states, None
@staticmethod
def _scatter_hidden_states(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
allow_reduce_scatter: bool = False,
*,
mhc: MHCState,
is_last_layer: bool,
**kwargs,
):
hidden_states, global_hidden_states = (
get_local_dp_buffer_mhc(get_parallel().tp_group, 1),
hidden_states,
)
# MoE skips its post-expert all-reduce with reduce_scatterv, so this
# scatter must reduce while combining local-expert partial sums.
if should_use_dp_reduce_scatterv():
get_parallel().tp_group.reduce_scatterv(
global_hidden_states,
output=hidden_states,
sizes=get_dp_global_num_tokens(),
)
elif allow_reduce_scatter and forward_batch.dp_padding_mode.is_max_len():
dp_reduce_scatter_tensor(hidden_states, global_hidden_states)
else:
dp_scatter(hidden_states, global_hidden_states, forward_batch)
hidden_states = mhc.mlp_combine(hidden_states, residual)
if not is_last_layer:
return hidden_states, None
hidden_states = hc_contract(hidden_states, mhc.hc_mult)
return hidden_states, None
@staticmethod
def _gather(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
*,
mhc: MHCState,
is_last_layer: bool,
**kwargs,
):
hidden_states = mhc.mlp_combine(hidden_states, residual)
if is_last_layer:
hidden_states = hc_contract(hidden_states, mhc.hc_mult)
hidden_states, local_hidden_states = (
get_local_dp_buffer_mhc(
get_parallel().tp_group, 1 if is_last_layer else mhc.hc_mult
),
hidden_states,
)
attn_tp_all_gather_into_tensor(hidden_states, local_hidden_states)
return hidden_states, None
@staticmethod
def _scatter(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
*,
mhc: MHCState,
is_last_layer: bool,
**kwargs,
):
hidden_states = hidden_states.tensor_split(context.attn_tp_size)[
context.attn_tp_rank
]
residual = residual.tensor_split(context.attn_tp_size)[context.attn_tp_rank]
hidden_states = mhc.mlp_combine(hidden_states, residual)
return hidden_states, None
class MHCLayerCommunicator(LayerCommunicator):
# Chooses its own boundary steps, not from the declarations.
_takes_declared_boundaries = False
"""A layer whose residual is hyper-connection streams: the shared boundary
steps, run with MHCState's residual operations."""
def __init__(
self,
@@ -395,158 +155,50 @@ class MHCLayerCommunicator(LayerCommunicator):
allow_reduce_scatter: bool = False,
qkv_latent_func: Optional[Callable] = None,
*,
is_first_layer: bool,
hc_mult: int,
hc_attn_pre: Callable,
hc_ffn_pre: Callable,
hc_post: Callable,
hc_ffn_post_pre: Optional[Callable] = None,
):
self.is_first_layer = is_first_layer
self.mhc = MHCState(
hc_mult=hc_mult,
hc_attn_pre=hc_attn_pre,
hc_ffn_pre=hc_ffn_pre,
hc_post=hc_post,
hc_ffn_post_pre=hc_ffn_post_pre,
is_last_layer=layer_scatter_modes.is_last_layer,
)
# The postprocess writes the FFN output into the streams, so the FFN's
# sum never waits for the next layer.
super().__init__(
layer_scatter_modes,
input_layernorm,
post_attention_layernorm,
allow_reduce_scatter,
qkv_latent_func,
allow_deferred_ffn_reduction=False,
residual_ops=self.mhc,
)
def _steps_from_declarations(
self, sides: DecoderLayerSides, **kwargs
) -> BoundarySteps:
# MHC has not been run with an FFN input gathered over attention CP.
if (
TokenAxis.ATTN_CP
in sides.attention_output.layout.sharded - sides.ffn.layout.sharded
):
raise NotImplementedError(
f"MHCLayerCommunicator with a gather over attention CP: {sides=}"
)
return super()._steps_from_declarations(sides, **kwargs)
def _post_init_communicate(self):
# Base MOE_FULL callables do not accept ``mhc``, so reject this
# combination at construction.
if self.layer_scatter_modes.mlp_mode == ScatterMode.MOE_FULL:
raise NotImplementedError(
"MHCLayerCommunicator does not support MOE_FULL "
"(moe_dp_size < attention_context_parallel_size). Increase "
"moe_dp_size to match attention_context_parallel_size."
)
self._communicate_simple_fn = CommunicateSimpleFn.get_fn(
input_mode=self.layer_scatter_modes.layer_input_mode,
output_mode=self.layer_scatter_modes.attn_mode,
context=self._context,
)
self._communicate_summable_tensor_pair_fn = (
MHCCommunicateSummableTensorPairFn.get_fn(
hidden_states_input_mode=self.layer_scatter_modes.mlp_mode,
residual_input_mode=self.layer_scatter_modes.middle_residual_mode,
output_mode=self.layer_scatter_modes.layer_output_mode,
context=self._context,
)
)
# MHC's own prepare_attn and postprocess run these.
return self._communicate_simple_fn, self._communicate_summable_tensor_pair_fn
def prepare_attn(
self,
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
):
self.publish_attn_lora_layout()
if self.is_first_layer:
if get_attn_tp_context().input_scattered:
hidden_states, _ = tp_reduce_scatter(
hidden_states,
None,
self._context,
)
hidden_states = hc_expand(hidden_states, self.mhc.hc_mult)
hidden_states, residual = self.mhc.attn_split(
hidden_states, out_norm=self.input_layernorm
)
hidden_states = self._communicate_simple_fn(
hidden_states=hidden_states,
forward_batch=forward_batch,
context=self._context,
)
# DSA and attention without a QKV hook consume full hidden states, so
# gather them before attention.
ctx = get_attn_tp_context()
dsa_pre_gather = ctx.input_scattered and ctx.is_dsa
no_qkv_latent_pre_gather = ctx.input_scattered and self.qkv_latent_func is None
if dsa_pre_gather or no_qkv_latent_pre_gather:
hidden_states = tp_all_gather_hidden_states(hidden_states, forward_batch)
if self.qkv_latent_func is not None:
attn_inputs = AttentionInputs(
hidden_states,
forward_batch,
self.qkv_latent_func,
is_pre_gathered=dsa_pre_gather,
)
ctx.set_attn_inputs(attn_inputs)
return hidden_states, residual
def _select_mlp_input(self):
"""MHC's own implementation of each boundary kind, applied to this
layer's MHC state."""
kind = mlp_input_kind(self.layer_scatter_modes, self._context)
residual_input_mode = self.layer_scatter_modes.layer_input_mode
fns = MHCCommunicateWithAllReduceAndLayerNormFn
if kind is MlpInputKind.NORM:
return partial(fns._simple, mhc=self.mhc), ()
if kind is MlpInputKind.GATHER:
fn = fns._gather_hidden_states_and_residual
elif kind is MlpInputKind.SCATTER:
fn = fns._scatter_hidden_states_and_residual
else:
raise NotImplementedError(f"MHCLayerCommunicator does not support {kind}")
return partial(fn, residual_input_mode=residual_input_mode, mhc=self.mhc), ()
def postprocess_layer(self, hidden_states, residual, forward_batch):
hidden_states, residual = self._communicate_summable_tensor_pair_fn(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
context=self._context,
allow_reduce_scatter=self.allow_reduce_scatter,
mhc=self.mhc,
is_last_layer=self.is_last_layer,
)
self.mhc.reset_aux()
return hidden_states, residual
def _select_ffn_completion(self, forward_batch: ForwardBatch) -> FfnCompletion:
"""An MHC layer's own postprocess completes its FFN output, combining the
hyper-connection streams; nothing is left to the next layer."""
return FfnCompletion(
defer_moe_finalize=False,
fuse_mlp_allreduce=False,
mlp_reduce_scatter=self.should_use_reduce_scatter(forward_batch),
complete=partial(
self._complete_ffn_output_now,
forward_batch=forward_batch,
dp_step=None,
),
)
def should_use_reduce_scatter(self, forward_batch: ForwardBatch):
if not self.allow_reduce_scatter:
return False
if (
self._communicate_summable_tensor_pair_fn
is MHCCommunicateSummableTensorPairFn._scatter_hidden_states
):
# reduce_scatterv already combines expert outputs; returning False
# would make RowParallelLinear perform an extra all-reduce.
if should_use_dp_reduce_scatterv():
return True
if forward_batch.dp_padding_mode.is_max_len():
return True
if get_attn_tp_context().input_scattered:
return True
return False
return super()._post_init_communicate()
+11 -22
View File
@@ -258,27 +258,14 @@ class _DpGatheredBufferWrapper:
return buffer
@classmethod
def get_local_dp_buffer(cls, group: GroupCoordinator) -> torch.Tensor:
dp = get_flags().dp
with use_symmetric_memory(group, disabled=not cls._dp_max_padding):
buffer = torch.empty(
(cls._local_dp_buffer_len, dp.buffer_hidden_size),
dtype=dp.buffer_dtype,
device=dp.buffer_device,
)
return buffer
@classmethod
def get_local_dp_buffer_mhc(
cls, group: GroupCoordinator, n: int = 1
def get_local_dp_buffer(
cls, group: GroupCoordinator, hidden_size: Optional[int] = None
) -> torch.Tensor:
from sglang.srt.runtime_context import get_flags
dp = get_flags().dp
with use_symmetric_memory(group, disabled=not cls._dp_max_padding):
buffer = torch.empty(
(cls._local_dp_buffer_len, dp.buffer_hidden_size * n),
(cls._local_dp_buffer_len, hidden_size or dp.buffer_hidden_size),
dtype=dp.buffer_dtype,
device=dp.buffer_device,
)
@@ -344,12 +331,14 @@ def get_global_dp_buffer(group: GroupCoordinator) -> torch.Tensor:
return _DpGatheredBufferWrapper.get_global_dp_buffer(group=group)
def get_local_dp_buffer(group: GroupCoordinator) -> torch.Tensor:
return _DpGatheredBufferWrapper.get_local_dp_buffer(group=group)
def get_local_dp_buffer_mhc(group: GroupCoordinator, n: int = 1) -> torch.Tensor:
return _DpGatheredBufferWrapper.get_local_dp_buffer_mhc(group=group, n=n)
def get_local_dp_buffer(
group: GroupCoordinator, hidden_size: Optional[int] = None
) -> torch.Tensor:
"""A buffer for this rank's local DP rows, ``hidden_size`` wide (the model's
hidden size by default)."""
return _DpGatheredBufferWrapper.get_local_dp_buffer(
group=group, hidden_size=hidden_size
)
def get_global_dp_buffer_len() -> int:
-1
View File
@@ -795,7 +795,6 @@ class Glm5NextDecoderLayer(nn.Module):
if self.config.mhc:
mhc_kwargs: Dict[str, Any] = dict(
is_first_layer=(self.layer_id == 0),
hc_mult=config.hc_mult,
hc_attn_pre=self.hc_attn_pre,
hc_ffn_pre=self.hc_ffn_pre,