[Refactor] Choose prepare_attn / prepare_mlp steps and fused kernels at construction (#41252)

This commit is contained in:
Cheng Wan
2026-09-26 02:38:03 -07:00
committed by GitHub
parent 7bc988446d
commit 8fc3ce48da
10 changed files with 765 additions and 581 deletions
+420 -353
View File
@@ -741,6 +741,8 @@ class LayerCommunicator:
self._communicate_summable_tensor_pair_fn
is CommunicateSummableTensorPairFn._scatter_hidden_states
)
self._mlp_input, self._mlp_input_may_fuse = self._select_mlp_input()
self._attn_input_fusions = self._select_attn_input_fusions()
self._speculative_algo = SpeculativeAlgorithm.from_string(
get_spec().speculative_algorithm
)
@@ -779,15 +781,6 @@ class LayerCommunicator:
output_mode=self.layer_scatter_modes.attn_mode,
context=self._context,
)
self._communicate_with_all_reduce_and_layer_norm_fn = (
CommunicateWithAllReduceAndLayerNormFn.get_fn(
hidden_states_input_mode=self.layer_scatter_modes.attn_mode,
residual_input_mode=self.layer_scatter_modes.layer_input_mode,
hidden_states_output_mode=self.layer_scatter_modes.mlp_mode,
residual_output_mode=self.layer_scatter_modes.middle_residual_mode,
context=self._context,
)
)
self._communicate_summable_tensor_pair_fn = (
CommunicateSummableTensorPairFn.get_fn(
hidden_states_input_mode=self.layer_scatter_modes.mlp_mode,
@@ -833,23 +826,14 @@ class LayerCommunicator:
"""True if ``prepare_mlp``'s post-attention RMSNorm leaves ``residual``
untouched, so Eagle3 aux capture can keep its reference and skip the clone.
Only the flashinfer all-reduce-fusion path writes a fresh ``residual_out``
(see ``flashinfer_allreduce_residual_rmsnorm``); the aiter fused kernel and
every plain norm fold into ``residual`` in place. That path is reachable
only from the ``_gather_*`` communicate-fns, and only when they fall past
their input-scattered branch.
Only the flashinfer all-reduce-fusion kernel writes a fresh
``residual_out`` (see ``flashinfer_allreduce_residual_rmsnorm``); the aiter
fused kernel and every plain norm fold into ``residual`` in place. It runs
when ``prepare_mlp`` selected its fused entry and the batch is not
input-scattered.
"""
norm_fn = getattr(
self._communicate_with_all_reduce_and_layer_norm_fn,
"func",
self._communicate_with_all_reduce_and_layer_norm_fn,
)
uses_gather_norm = norm_fn in (
CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual,
CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual_moe,
)
return (
uses_gather_norm
self._mlp_input_may_fuse
and not get_attn_tp_context().input_scattered
and apply_flashinfer_allreduce_fusion(residual.shape[0])
)
@@ -881,22 +865,28 @@ class LayerCommunicator:
):
self.publish_attn_lora_layout()
if isinstance(hidden_states, UnreducedOutput) and residual is None:
raise RuntimeError("an UnreducedOutput requires residual input")
# What the previous layer left to complete: an UnreducedOutput, or a
# producer's handoff that one of the fused entries consumes.
owed = None if isinstance(hidden_states, torch.Tensor) else hidden_states
if owed is not None and residual is None:
raise RuntimeError(f"{type(owed).__name__} requires residual input")
if (
isinstance(hidden_states, UnreducedOutput)
and hidden_states.reduce_and_redistribute is not None
isinstance(owed, UnreducedOutput)
and owed.reduce_and_redistribute is not None
):
# No fused kernel runs under attention DP: the reduce-scatter back to
# this rank's tokens comes first.
hidden_states = reduce_output(hidden_states)
unreduced = (
hidden_states if isinstance(hidden_states, UnreducedOutput) else None
)
pending = unreduced is not None
if pending:
hidden_states = unreduced.partial
hidden_states, owed = reduce_output(owed), None
if owed is not None:
for fused in self._attn_input_fusions:
result = fused(owed, residual, forward_batch, post_residual_addition)
if result is not None:
return self._finish_prepare_attn(
hidden_states=result[0],
residual=result[1],
forward_batch=forward_batch,
)
hidden_states = owed.partial
# residual is None marks the first decoder layer, where the SP region
# opens: re-evaluated per forward so a crash mid-loop cannot leak into
# the next one.
@@ -922,23 +912,11 @@ class LayerCommunicator:
)
if hidden_states.shape[0] == 0:
residual = hidden_states
elif residual is not None and pending:
fused = (
self._reduce_output_and_update_and_read_residual(
hidden_states, residual, forward_batch
)
if post_residual_addition is None
# The fused kernel reduces over the MoE output's group.
and unreduced.group is post_experts_reduction_group()
else None
)
if fused is not None:
hidden_states, residual = fused
else:
hidden_states = reduce_output(unreduced)
hidden_states, residual = _attn_input_update_and_read_residual(
quant_format
)(self.input_layernorm, hidden_states, residual, post_residual_addition)
elif owed is not None:
hidden_states = reduce_output(owed)
hidden_states, residual = _attn_input_update_and_read_residual(
quant_format
)(self.input_layernorm, hidden_states, residual, post_residual_addition)
else:
hidden_states, residual = _attn_input_update_and_read_residual(
quant_format
@@ -964,43 +942,63 @@ class LayerCommunicator:
get_attn_tp_context().set_attn_inputs(attn_inputs)
return hidden_states, residual
def _select_attn_input_fusions(self) -> Tuple[Callable, ...]:
"""The fused kernels that complete what the previous layer left together
with the residual update and the input norm, in the order they are tried.
Each takes (owed, residual, forward_batch, post_residual_addition) and
returns None when it does not take the batch."""
if not hasattr(self.input_layernorm, "forward_with_allreduce_fusion"):
return ()
self._attn_input_fuses_quant = (
self.enable_fused_ar_quant
and _use_aiter
and hasattr(
self.input_layernorm, "forward_with_allreduce_fusion_quant_per_group"
)
)
return (self._reduce_output_and_update_and_read_residual,)
def _reduce_output_and_update_and_read_residual(
self,
hidden_states: torch.Tensor,
owed: UnreducedOutput,
residual: torch.Tensor,
forward_batch: ForwardBatch,
post_residual_addition: Optional[torch.Tensor],
) -> Optional[Tuple[torch.Tensor, torch.Tensor]]:
"""Complete the sum the previous layer left, add it to the residual and
apply the input norm in one fused kernel; None when the kernel does not
take this batch. The result is not quantized for ``quant_format``."""
apply the input norm in one aiter or flashinfer kernel; None when the
kernel does not take this batch. The result is not quantized for
``quant_format``."""
if (
not isinstance(owed, UnreducedOutput)
# The kernel does not add it.
or post_residual_addition is not None
# The kernel reduces over the MoE output's group.
or owed.group is not post_experts_reduction_group()
):
return None
hidden_states = owed.partial
if not (
apply_aiter_all_reduce_fusion(hidden_states, forward_batch)
or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
) and hasattr(self.input_layernorm, "forward_with_allreduce_fusion"):
if (
self.enable_fused_ar_quant
and _use_aiter
and hasattr(
self.input_layernorm,
"forward_with_allreduce_fusion_quant_per_group",
):
return None
if self._attn_input_fuses_quant:
# Falls back to AR+RMSNorm + separate quant internally when the
# fully-fused kernel cannot service the shape.
quant_result = (
self.input_layernorm.forward_with_allreduce_fusion_quant_per_group(
hidden_states,
residual,
use_attn_tp_group=False,
keep_bf16=self.fused_ar_quant_keep_bf16,
)
):
# Falls back to AR+RMSNorm + separate quant internally when the
# fully-fused kernel cannot service the shape.
quant_result = (
self.input_layernorm.forward_with_allreduce_fusion_quant_per_group(
hidden_states,
residual,
use_attn_tp_group=False,
keep_bf16=self.fused_ar_quant_keep_bf16,
)
)
if quant_result is not None:
return quant_result
return self.input_layernorm.forward_with_allreduce_fusion(
hidden_states, residual, use_attn_tp_group=False
)
return None
if quant_result is not None:
return quant_result
return self.input_layernorm.forward_with_allreduce_fusion(
hidden_states, residual, use_attn_tp_group=False
)
def _tp_reduce_scatter(
self,
@@ -1009,6 +1007,69 @@ class LayerCommunicator:
) -> Tuple[torch.Tensor, torch.Tensor]:
return tp_reduce_scatter(hidden_states, residual, self._context)
def _select_mlp_input(self) -> Tuple[Callable, bool]:
"""The attention-TP -> FFN boundary's steps, chosen from the layouts, and
whether they include the aiter / flashinfer fused entry. The MHC and
DSA-CP communicators pick their own implementations."""
kind = mlp_input_kind(self.layer_scatter_modes, self._context)
residual_input_mode = self.layer_scatter_modes.layer_input_mode
if kind is MlpInputKind.NORM:
return _mlp_input_norm, False
if kind is MlpInputKind.ATTN_TP_ALL_REDUCE:
return _mlp_input_attn_tp_all_reduce, False
if kind is MlpInputKind.SCATTER:
return (
partial(
_mlp_input_scatter,
scatters_residual=residual_input_mode == ScatterMode.TP_ATTN_FULL,
),
False,
)
# Neither fused kernel runs under attention DP.
fusions = (
self._select_mlp_input_fusions(residual_input_mode)
if self._context.attn_dp_size == 1
else ()
)
steps = partial(
_mlp_input_gather,
order=_mlp_input_order(self._context, residual_input_mode, fusions),
)
if kind is MlpInputKind.GATHER_MOE_CP:
steps = partial(_mlp_input_gather_moe_cp, gather=steps)
return (
steps,
self._mlp_input_reduce_output_and_update_and_read_residual in fusions,
)
def _select_mlp_input_fusions(
self, residual_input_mode: ScatterMode
) -> Tuple[Callable, ...]:
"""The fused kernels that complete the attention output's all-reduce
together with the residual update and the post-attention norm, in the
order they are tried. Each takes (hidden_states, residual, forward_batch)
and returns None when it does not take the batch."""
if not hasattr(self.post_attention_layernorm, "forward_with_allreduce_fusion"):
return ()
return (self._mlp_input_reduce_output_and_update_and_read_residual,)
def _mlp_input_reduce_output_and_update_and_read_residual(
self,
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
) -> Optional[Tuple[torch.Tensor, torch.Tensor]]:
"""The attention-TP all-reduce, residual add and norm in one aiter or
flashinfer kernel; None when neither takes the batch."""
if not (
apply_aiter_all_reduce_fusion(hidden_states, forward_batch)
or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
):
return None
return self.post_attention_layernorm.forward_with_allreduce_fusion(
hidden_states, residual, use_attn_tp_group=True
)
def prepare_mlp(
self,
hidden_states: torch.Tensor,
@@ -1024,12 +1085,12 @@ class LayerCommunicator:
if cache is not None:
self._context.cache = cache
return self._communicate_with_all_reduce_and_layer_norm_fn(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
layernorm=self.post_attention_layernorm,
context=self._context,
return self._mlp_input(
hidden_states,
residual,
forward_batch,
self.post_attention_layernorm,
self._context,
)
def maybe_prefetch_next_full_attention_kv(
@@ -1515,25 +1576,6 @@ def _mlp_input_reduce_output(
return attention_tensor_model_parallel_all_reduce(hidden_states)
def _mlp_input_reduce_output_and_update_and_read_residual(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
) -> Optional[Tuple[torch.Tensor, torch.Tensor]]:
"""The attention-TP all-reduce, residual add and norm in one fused kernel;
None when no fused kernel takes the batch."""
if (
apply_aiter_all_reduce_fusion(hidden_states, forward_batch)
or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
) and hasattr(layernorm, "forward_with_allreduce_fusion"):
hidden_states, residual = layernorm.forward_with_allreduce_fusion(
hidden_states, residual, use_attn_tp_group=True
)
return hidden_states, residual
return None
def _redistribute_input_to_dp(
hidden_states: torch.Tensor, forward_batch: ForwardBatch
) -> torch.Tensor:
@@ -1550,259 +1592,284 @@ def _reduce_and_redistribute_output_to_dp(
return global_hidden_states
class CommunicateWithAllReduceAndLayerNormFn:
"""Besides communication, needs to
1. All reduce in tp_attn_group on hidden_states
2. Apply layer norm
def _mlp_input_without_dp(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
gathers_residual: bool,
fusions: Tuple[Callable, ...],
):
if gathers_residual:
residual = _redistribute_from_attn_tp_shards(residual)
for fused in fusions:
result = fused(hidden_states, residual, forward_batch)
if result is not None:
return result
hidden_states = _mlp_input_reduce_output(hidden_states, forward_batch)
if _is_npu and context.cache is not None:
_ = prepare_weight_cache(hidden_states, context.cache)
return layernorm(hidden_states, residual)
def _mlp_input_dp_replicate(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
gathers_residual: bool,
):
"""Attention DP: reduce, add and normalize locally, then gather."""
if gathers_residual:
residual = _redistribute_from_attn_tp_shards(residual)
if hidden_states.shape[0] != 0:
if context.attn_tp_size > 1:
hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states)
with use_symmetric_memory(
get_parallel().tp_group,
disabled=not is_allocation_symmetric(),
):
hidden_states, residual = layernorm(hidden_states, residual)
return _redistribute_input_to_dp(hidden_states, forward_batch), residual
def _mlp_input_dp_partial(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
gathers_residual: bool,
):
"""Attention DP: one rank adds the residual, the gather sums it, then
normalize."""
if gathers_residual:
residual = _redistribute_from_attn_tp_shards(residual)
if context.attn_tp_rank == 0:
hidden_states += residual
hidden_states = _reduce_and_redistribute_output_to_dp(hidden_states, forward_batch)
dp_scatter(residual, hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = layernorm(hidden_states)
return hidden_states, residual
def _mlp_input_order(
context: CommunicateContext,
residual_input_mode: ScatterMode,
fusions: Tuple[Callable, ...],
) -> Callable:
"""The steps from the attention output to the FFN input, chosen from facts
fixed at construction."""
gathers_residual = (
residual_input_mode == ScatterMode.SCATTERED and context.attn_tp_size > 1
)
if context.attn_dp_size == 1:
return partial(
_mlp_input_without_dp, gathers_residual=gathers_residual, fusions=fusions
)
if context.force_layernorm_before_dp_gather or context.attn_tp_size == 1:
return partial(_mlp_input_dp_replicate, gathers_residual=gathers_residual)
return partial(_mlp_input_dp_partial, gathers_residual=gathers_residual)
class MlpInputKind(Enum):
"""What the attention-TP -> FFN boundary does, given the two sides' layouts."""
# The layouts agree: add the residual and normalize.
NORM = auto()
# All-reduce over attention TP, normalize, and gather for the FFN group.
GATHER = auto()
# The same, then gather over the MoE-CP group (moe_dp_size < attn_cp_size).
GATHER_MOE_CP = auto()
# Reduce-scatter over attention TP to this rank's tokens, then normalize.
SCATTER = auto()
# All-reduce over attention TP for a dense MLP on that group.
ATTN_TP_ALL_REDUCE = auto()
def mlp_input_kind(
modes: LayerScatterModes, context: CommunicateContext
) -> MlpInputKind:
hidden_in, residual_in = modes.attn_mode, modes.layer_input_mode
hidden_out, residual_out = modes.mlp_mode, modes.middle_residual_mode
if (
context.is_same_layout(hidden_in, hidden_out)
and context.is_same_layout(residual_in, residual_out)
and context.attn_tp_size == 1
):
return MlpInputKind.NORM
if (
hidden_in == ScatterMode.SCATTERED
and residual_in == ScatterMode.SCATTERED
and hidden_out == ScatterMode.SCATTERED
and residual_out == ScatterMode.SCATTERED
):
# Megatron LayerNorm sequence parallelism (layers/layernorm_sp.py):
# activations stay sequence-sharded across the attn->mlp boundary, so
# there is nothing to gather or scatter here -- just the residual add
# plus LayerNorm on the local shard. The row-parallel o_proj already
# issued the reduce-scatter (g-bar) that the all-reduce would have
# done, and the g all-gather is fused into the next column-parallel
# linear. Distinct from the branch above because under pure TP
# attn_tp_size == tp_size > 1, so that gate does not fire.
return MlpInputKind.NORM
if hidden_in == ScatterMode.TP_ATTN_FULL and residual_in in (
ScatterMode.SCATTERED,
ScatterMode.TP_ATTN_FULL,
):
kind = {
(ScatterMode.FULL, ScatterMode.TP_ATTN_FULL): MlpInputKind.GATHER,
(
ScatterMode.MOE_FULL,
ScatterMode.TP_ATTN_FULL,
): MlpInputKind.GATHER_MOE_CP,
(ScatterMode.SCATTERED, ScatterMode.SCATTERED): MlpInputKind.SCATTER,
}.get((hidden_out, residual_out))
if kind is not None:
return kind
if (
hidden_out == ScatterMode.TP_ATTN_FULL
and residual_out == ScatterMode.TP_ATTN_FULL
and context.attn_tp_size > 1
):
# Used when the dense MLP is tensor-parallelized along the
# attention TP group (``moe_dense_tp_size > 1``): hidden states
# need an all-reduce inside the attention TP group before the
# next layernorm, while staying in TP_ATTN_FULL on both sides.
return MlpInputKind.ATTN_TP_ALL_REDUCE
raise NotImplementedError(
f"{hidden_in=} {residual_in=} {hidden_out=} {residual_out=}"
)
def _mlp_input_norm(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
):
# TODO move these `if shape != 0` into LayerNorm itself
if hidden_states.shape[0] != 0:
hidden_states, residual = layernorm(hidden_states, residual)
return hidden_states, residual
def _mlp_input_attn_tp_all_reduce(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
):
"""All-reduce hidden states inside the attention TP group, then layernorm.
Used when the dense MLP shares the attention TP group
(``moe_dense_tp_size > 1``): both hidden states and residual stay in
``TP_ATTN_FULL`` across the boundary.
"""
hidden_states = get_parallel().attn_tp_group.all_reduce(hidden_states)
if hidden_states.shape[0] != 0:
hidden_states, residual = layernorm(hidden_states, residual)
return hidden_states, residual
@staticmethod
def get_fn(
hidden_states_input_mode: ScatterMode,
residual_input_mode: ScatterMode,
hidden_states_output_mode: ScatterMode,
residual_output_mode: ScatterMode,
context: CommunicateContext,
def _mlp_input_scatter(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
scatters_residual: bool,
):
hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards(
hidden_states, context
)
if scatters_residual:
residual = _redistribute_to_attn_tp_shards(residual, context)
if hidden_states.shape[0] != 0:
hidden_states, residual = layernorm(hidden_states, residual)
return hidden_states, residual
def _tp_all_reduce_with_scattered_residual(
hidden_states: torch.Tensor,
residual: torch.Tensor,
layernorm: torch.nn.Module,
context: CommunicateContext,
):
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
scattered_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
scattered_states += residual
residual = tensor_model_parallel_all_reduce(hidden_states)
hidden_states = layernorm(residual)
return hidden_states, residual
def _mlp_input_gather(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
order: Callable,
):
"""Run ``order``, the steps ``_mlp_input_order`` chose at construction,
unless this batch's attention input is scattered."""
if get_attn_tp_context().input_scattered:
return _tp_all_reduce_with_scattered_residual(
hidden_states, residual, layernorm, context
)
return order(hidden_states, residual, forward_batch, layernorm, context)
def _mlp_input_gather_moe_cp(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
gather: Callable,
):
"""Gather for the FFN, then over the MoE-CP group so each rank holds all
tokens of its MoE group (moe_dp_size < attn_cp_size). The residual stays at
TP_ATTN_FULL."""
# Early return on empty tensor is safe for MOE_CP because:
# - During CP extend: zigzag split guarantees all CP ranks have non-zero tokens,
# so no rank hits this path while others proceed to the allgather.
# - During decode: moe_cp allgather is skipped (guarded by is_context_parallel_extend).
# - CUDA graph warmup: not applicable when --cuda-graph-backend-prefill=disabled is used.
if hidden_states.shape[0] == 0:
return hidden_states, residual
hidden_states, residual = gather(
hidden_states, residual, forward_batch, layernorm, context
)
# Only active during prefill (context-parallel extend); decode keeps existing path.
moe_cp_size = get_moe_cp_size()
if (
moe_cp_size > 1
and hidden_states.shape[0] > 0
and forward_batch.forward_mode.is_context_parallel_extend()
and forward_batch.attn_cp_metadata is not None
):
if (
context.is_same_layout(hidden_states_input_mode, hidden_states_output_mode)
and context.is_same_layout(residual_input_mode, residual_output_mode)
and context.attn_tp_size == 1
):
return CommunicateWithAllReduceAndLayerNormFn._simple
if (
hidden_states_input_mode == ScatterMode.SCATTERED
and residual_input_mode == ScatterMode.SCATTERED
and hidden_states_output_mode == ScatterMode.SCATTERED
and residual_output_mode == ScatterMode.SCATTERED
):
# Megatron LayerNorm sequence parallelism (layers/layernorm_sp.py):
# activations stay sequence-sharded across the attn->mlp boundary, so
# there is nothing to gather or scatter here -- just the residual add
# plus LayerNorm on the local shard. The row-parallel o_proj already
# issued the reduce-scatter (g-bar) that the all-reduce would have
# done, and the g all-gather is fused into the next column-parallel
# linear. Distinct from the branch above because under pure TP
# attn_tp_size == tp_size > 1, so that gate does not fire.
return CommunicateWithAllReduceAndLayerNormFn._simple
if hidden_states_input_mode == ScatterMode.TP_ATTN_FULL and (
residual_input_mode in (ScatterMode.SCATTERED, ScatterMode.TP_ATTN_FULL)
):
fn = {
(
ScatterMode.FULL,
ScatterMode.TP_ATTN_FULL,
): CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual,
(
ScatterMode.MOE_FULL,
ScatterMode.TP_ATTN_FULL,
): CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual_moe,
(
ScatterMode.SCATTERED,
ScatterMode.SCATTERED,
): CommunicateWithAllReduceAndLayerNormFn._scatter_hidden_states_and_residual,
}.get((hidden_states_output_mode, residual_output_mode))
if fn is not None:
return partial(fn, residual_input_mode=residual_input_mode)
if (
hidden_states_output_mode == ScatterMode.TP_ATTN_FULL
and residual_output_mode == ScatterMode.TP_ATTN_FULL
and context.attn_tp_size > 1
):
# Used when the dense MLP is tensor-parallelized along the
# attention TP group (``moe_dense_tp_size > 1``): hidden states
# need an all-reduce inside the attention TP group before the
# next layernorm, while staying in TP_ATTN_FULL on both sides.
return CommunicateWithAllReduceAndLayerNormFn._tp_attn_all_reduce_and_layernorm
raise NotImplementedError(
f"{hidden_states_input_mode=} {residual_input_mode=} {hidden_states_output_mode=} {residual_output_mode=}"
hidden_states = _redistribute_input_to_moe_cp(
hidden_states, forward_batch, moe_cp_size
)
@staticmethod
def _simple(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
):
# TODO move these `if shape != 0` into LayerNorm itself
if hidden_states.shape[0] != 0:
hidden_states, residual = layernorm(hidden_states, residual)
return hidden_states, residual
@staticmethod
def _tp_attn_all_reduce_and_layernorm(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
):
"""All-reduce hidden states inside the attention TP group, then layernorm.
Used when the dense MLP shares the attention TP group
(``moe_dense_tp_size > 1``): both hidden states and residual stay in
``TP_ATTN_FULL`` across the boundary.
"""
hidden_states = get_parallel().attn_tp_group.all_reduce(hidden_states)
if hidden_states.shape[0] != 0:
hidden_states, residual = layernorm(hidden_states, residual)
return hidden_states, residual
@staticmethod
def _gather_hidden_states_and_residual(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
residual_input_mode,
):
if get_attn_tp_context().input_scattered:
return CommunicateWithAllReduceAndLayerNormFn._tp_all_reduce_with_scattered_residual(
hidden_states,
residual,
layernorm,
context,
)
if residual_input_mode == ScatterMode.SCATTERED and context.attn_tp_size > 1:
residual = _redistribute_from_attn_tp_shards(residual)
if context.attn_dp_size == 1:
fused = _mlp_input_reduce_output_and_update_and_read_residual(
hidden_states, residual, forward_batch, layernorm
)
if fused is not None:
return fused
hidden_states = _mlp_input_reduce_output(hidden_states, forward_batch)
if _is_npu and context.cache is not None:
_ = prepare_weight_cache(hidden_states, context.cache)
return layernorm(hidden_states, residual)
# Attention DP. Replicate: reduce, add and normalize locally, then gather.
# Partial: one rank adds the residual, the gather sums it, then normalize.
replicate = (
context.force_layernorm_before_dp_gather or context.attn_tp_size == 1
)
if replicate and hidden_states.shape[0] != 0:
if context.attn_tp_size > 1:
hidden_states = attention_tensor_model_parallel_all_reduce(
hidden_states
)
with use_symmetric_memory(
get_parallel().tp_group,
disabled=not is_allocation_symmetric(),
):
hidden_states, residual = layernorm(hidden_states, residual)
elif context.attn_tp_rank == 0:
hidden_states += residual
if replicate:
return _redistribute_input_to_dp(hidden_states, forward_batch), residual
hidden_states = _reduce_and_redistribute_output_to_dp(
hidden_states, forward_batch
)
dp_scatter(residual, hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = layernorm(hidden_states)
return hidden_states, residual
@staticmethod
def _scatter_hidden_states_and_residual(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
residual_input_mode,
):
hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards(
hidden_states, context
)
if residual_input_mode == ScatterMode.TP_ATTN_FULL:
residual = _redistribute_to_attn_tp_shards(residual, context)
if hidden_states.shape[0] != 0:
hidden_states, residual = layernorm(hidden_states, residual)
return hidden_states, residual
@staticmethod
def _tp_all_reduce_with_scattered_residual(
hidden_states: torch.Tensor,
residual: torch.Tensor,
layernorm: torch.nn.Module,
context: CommunicateContext,
):
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
scattered_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
scattered_states += residual
residual = tensor_model_parallel_all_reduce(hidden_states)
hidden_states = layernorm(residual)
return hidden_states, residual
@staticmethod
def _gather_hidden_states_and_residual_moe(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
residual_input_mode,
):
"""Allgather tokens for MoE when moe_dp_size < attn_cp_size.
Steps:
1. Standard attn-TP all-reduce + optional DP allgather + layernorm (same as
_gather_hidden_states_and_residual for the dp>1 case, or simple all-reduce
+ layernorm for dp==1).
2. moe_cp allgather: gather tokens from cp_per_moe CP ranks so each rank holds
all tokens for its MoE group.
Residual is left at TP_ATTN_FULL throughout.
"""
# Early return on empty tensor is safe for MOE_CP because:
# - During CP extend: zigzag split guarantees all CP ranks have non-zero tokens,
# so no rank hits this path while others proceed to the allgather.
# - During decode: moe_cp allgather is skipped (guarded by is_context_parallel_extend).
# - CUDA graph warmup: not applicable when --cuda-graph-backend-prefill=disabled is used.
if hidden_states.shape[0] == 0:
return hidden_states, residual
# Step 1: Standard all-reduce/DP-allgather + layernorm (reuse existing logic).
hidden_states, residual = (
CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
layernorm=layernorm,
context=context,
residual_input_mode=residual_input_mode,
)
)
# Step 2: moe_cp allgather — gather across cp_per_moe CP ranks.
# Only active during prefill (context-parallel extend); decode keeps existing path.
moe_cp_size = get_moe_cp_size()
if (
moe_cp_size > 1
and hidden_states.shape[0] > 0
and forward_batch.forward_mode.is_context_parallel_extend()
and forward_batch.attn_cp_metadata is not None
):
hidden_states = _redistribute_input_to_moe_cp(
hidden_states, forward_batch, moe_cp_size
)
return hidden_states, residual
return hidden_states, residual
def _dp_scatter_group() -> GroupCoordinator:
@@ -2022,7 +2089,7 @@ class CommunicateSummableTensorPairFn:
If DP>1, further scatter back to the local DP slice.
"""
# Only scatter back during prefill; decode was never allgathered so no-op.
# Safe w.r.t. empty tensors: same reasoning as _gather_hidden_states_and_residual_moe
# Safe w.r.t. empty tensors: same reasoning as _mlp_input_gather_moe_cp
# — CP extend always has non-zero tokens per rank, and decode skips this path.
moe_cp_size = get_moe_cp_size()
if (
+13 -12
View File
@@ -25,10 +25,10 @@ from sglang.srt.layers.communicator import (
CommunicateContext,
CommunicateSimpleFn,
CommunicateSummableTensorPairFn,
CommunicateWithAllReduceAndLayerNormFn,
LayerCommunicator,
LayerScatterModes,
ScatterMode,
_mlp_input_norm,
)
from sglang.srt.layers.cp.utils import is_mla_cp_active
from sglang.srt.layers.dp_attention import (
@@ -113,13 +113,6 @@ class DSACPLayerCommunicator(LayerCommunicator):
output_mode=ScatterMode.SCATTERED,
context=self._context,
)
self._communicate_with_all_reduce_and_layer_norm_fn = DSACPCommunicateWithAllReduceAndLayerNormFn.get_fn(
hidden_states_input_mode=ScatterMode.SCATTERED,
residual_input_mode=ScatterMode.SCATTERED,
hidden_states_output_mode=self.layer_scatter_modes.mlp_mode, # SCATTERED, FULL
residual_output_mode=ScatterMode.SCATTERED,
context=self._context,
)
self._communicate_summable_tensor_pair_fn = DSACPCommunicateSummableTensorPairFn.get_fn(
hidden_states_input_mode=self.layer_scatter_modes.mlp_mode, # SCATTERED, FULL
residual_input_mode=ScatterMode.SCATTERED,
@@ -127,6 +120,16 @@ class DSACPLayerCommunicator(LayerCommunicator):
context=self._context,
)
def _select_mlp_input(self):
fn = DSACPCommunicateWithAllReduceAndLayerNormFn.get_fn(
hidden_states_input_mode=ScatterMode.SCATTERED,
residual_input_mode=ScatterMode.SCATTERED,
hidden_states_output_mode=self.layer_scatter_modes.mlp_mode, # SCATTERED, FULL
residual_output_mode=ScatterMode.SCATTERED,
context=self._context,
)
return fn, False
class DSACPCommunicateSimpleFn(CommunicateSimpleFn):
@staticmethod
@@ -141,9 +144,7 @@ class DSACPCommunicateSimpleFn(CommunicateSimpleFn):
raise NotImplementedError(f"{input_mode=} {output_mode=}")
class DSACPCommunicateWithAllReduceAndLayerNormFn(
CommunicateWithAllReduceAndLayerNormFn
):
class DSACPCommunicateWithAllReduceAndLayerNormFn:
"""Besides communication, needs to
1. All reduce in tp_attn_group on hidden_states
2. Apply layer norm
@@ -161,7 +162,7 @@ class DSACPCommunicateWithAllReduceAndLayerNormFn(
assert residual_input_mode == ScatterMode.SCATTERED
assert residual_output_mode == ScatterMode.SCATTERED
if hidden_states_output_mode == ScatterMode.SCATTERED:
return DSACPCommunicateWithAllReduceAndLayerNormFn._simple
return _mlp_input_norm
if hidden_states_output_mode == ScatterMode.FULL:
return partial(
+18 -60
View File
@@ -29,11 +29,12 @@ from sglang.srt.layers.communicator import (
CommunicateContext,
CommunicateSimpleFn,
CommunicateSummableTensorPairFn,
CommunicateWithAllReduceAndLayerNormFn,
LayerCommunicator,
LayerScatterModes,
MlpInputKind,
ScatterMode,
get_attn_tp_context,
mlp_input_kind,
tp_reduce_scatter,
)
from sglang.srt.layers.dp_attention import (
@@ -130,35 +131,7 @@ class MHCState:
self.h_post = None
class MHCCommunicateWithAllReduceAndLayerNormFn(CommunicateWithAllReduceAndLayerNormFn):
@staticmethod
def get_fn(
hidden_states_input_mode: ScatterMode,
residual_input_mode: ScatterMode,
hidden_states_output_mode: ScatterMode,
residual_output_mode: ScatterMode,
context: CommunicateContext,
):
fn = CommunicateWithAllReduceAndLayerNormFn.get_fn(
hidden_states_input_mode,
residual_input_mode,
hidden_states_output_mode,
residual_output_mode,
context,
)
replacements = {
CommunicateWithAllReduceAndLayerNormFn._simple: MHCCommunicateWithAllReduceAndLayerNormFn._simple,
CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual: MHCCommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual,
CommunicateWithAllReduceAndLayerNormFn._scatter_hidden_states_and_residual: MHCCommunicateWithAllReduceAndLayerNormFn._scatter_hidden_states_and_residual,
}
if isinstance(fn, partial):
return partial(
replacements.get(fn.func, fn.func),
*fn.args,
**(fn.keywords or {}),
)
return replacements.get(fn, fn)
class MHCCommunicateWithAllReduceAndLayerNormFn:
@staticmethod
def _scatter_hidden_states_and_residual(
hidden_states: torch.Tensor,
@@ -456,15 +429,6 @@ class MHCLayerCommunicator(LayerCommunicator):
output_mode=self.layer_scatter_modes.attn_mode,
context=self._context,
)
self._communicate_with_all_reduce_and_layer_norm_fn = (
MHCCommunicateWithAllReduceAndLayerNormFn.get_fn(
hidden_states_input_mode=self.layer_scatter_modes.attn_mode,
residual_input_mode=self.layer_scatter_modes.layer_input_mode,
hidden_states_output_mode=self.layer_scatter_modes.mlp_mode,
residual_output_mode=self.layer_scatter_modes.middle_residual_mode,
context=self._context,
)
)
self._communicate_summable_tensor_pair_fn = (
MHCCommunicateSummableTensorPairFn.get_fn(
hidden_states_input_mode=self.layer_scatter_modes.mlp_mode,
@@ -519,27 +483,21 @@ class MHCLayerCommunicator(LayerCommunicator):
return hidden_states, residual
def prepare_mlp(
self,
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
cache=None,
):
self.publish_mlp_lora_layout()
if cache is not None:
self._context.cache = cache
hidden_states, residual = self._communicate_with_all_reduce_and_layer_norm_fn(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
layernorm=self.post_attention_layernorm,
context=self._context,
mhc=self.mhc,
)
return hidden_states, residual
def _select_mlp_input(self):
"""MHC's own implementation of each boundary kind, applied to this
layer's MHC state."""
kind = mlp_input_kind(self.layer_scatter_modes, self._context)
residual_input_mode = self.layer_scatter_modes.layer_input_mode
fns = MHCCommunicateWithAllReduceAndLayerNormFn
if kind is MlpInputKind.NORM:
return partial(fns._simple, mhc=self.mhc), False
if kind is MlpInputKind.GATHER:
fn = fns._gather_hidden_states_and_residual
elif kind is MlpInputKind.SCATTER:
fn = fns._scatter_hidden_states_and_residual
else:
raise NotImplementedError(f"MHCLayerCommunicator does not support {kind}")
return partial(fn, residual_input_mode=residual_input_mode, mhc=self.mhc), False
def postprocess_layer(self, hidden_states, residual, forward_batch):
hidden_states, residual = self._communicate_summable_tensor_pair_fn(
+73 -100
View File
@@ -7,7 +7,6 @@ shared-expert add folded in when the runner hands back a MoeFinalizeHandoff.
from __future__ import annotations
import functools
import logging
from typing import Callable, Optional, Sequence
@@ -15,7 +14,6 @@ import msgspec
import torch
from sglang.srt.layers.communicator import (
CommunicateWithAllReduceAndLayerNormFn,
LayerCommunicator,
ScatterMode,
UnreducedOutput,
@@ -187,112 +185,87 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator):
# A replicated output follows the reduction; moving it would scale that by tp.
owes_local_reduction: bool = False
def prepare_attn(
self,
hidden_states,
residual,
forward_batch,
quant_format: str = "",
post_residual_addition=None,
):
if isinstance(hidden_states, MoeFinalizeHandoff):
if not self._should_use_finalize(forward_batch, hidden_states.m):
raise RuntimeError(
"received deferred MoE output on an ineligible path "
f"(M={hidden_states.m}, mode={forward_batch.forward_mode})"
)
if residual is None:
raise RuntimeError("deferred MoE finalize requires residual input")
gamma = _fused_norm_gamma(self.input_layernorm)
if gamma is None:
raise RuntimeError(
"deferred MoE finalize requires a fusable RMSNorm flavour"
)
if post_residual_addition is not None:
residual = residual + post_residual_addition
assert self.fusion_service is not None
hidden_states, residual = self.fusion_service.finalize(
handoff=hidden_states, residual=residual, gamma=gamma
)
return self._finish_prepare_attn(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
)
if (
residual is not None
and isinstance(hidden_states, UnreducedOutput)
and self._can_consume_post_moe_all_reduce(
forward_batch, int(hidden_states.partial.shape[0])
)
):
if post_residual_addition is not None:
residual = residual + post_residual_addition
assert self.fusion_service is not None
hidden_states, residual = self.fusion_service.all_reduce_residual_rms_norm(
local_contribution=hidden_states.partial,
residual=residual,
gamma=_fused_norm_gamma(self.input_layernorm),
)
return self._finish_prepare_attn(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
)
return super().prepare_attn(
hidden_states,
residual,
forward_batch,
quant_format=quant_format,
post_residual_addition=post_residual_addition,
def _select_attn_input_fusions(self):
return (
self._finalize_output_and_update_and_read_residual_cutedsl,
self._reduce_output_and_update_and_read_residual_cutedsl,
*super()._select_attn_input_fusions(),
)
def prepare_mlp(
self,
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
cache=None,
def _finalize_output_and_update_and_read_residual_cutedsl(
self, owed, residual, forward_batch, post_residual_addition
):
if cache is not None:
self._context.cache = cache
if self._should_use_all_reduce_rms_norm(
forward_batch, int(hidden_states.shape[0]), residual
):
assert self.fusion_service is not None and residual is not None
return self.fusion_service.all_reduce_residual_rms_norm(
local_contribution=hidden_states,
residual=residual,
gamma=_fused_norm_gamma(self.post_attention_layernorm),
"""Finish a deferred MoE finalize with the all-reduce, residual add and
input norm."""
if not isinstance(owed, MoeFinalizeHandoff):
return None
if not self._should_use_finalize(forward_batch, owed.m):
raise RuntimeError(
"received deferred MoE output on an ineligible path "
f"(M={owed.m}, mode={forward_batch.forward_mode})"
)
return super().prepare_mlp(hidden_states, residual, forward_batch, cache=cache)
gamma = _fused_norm_gamma(self.input_layernorm)
if gamma is None:
raise RuntimeError(
"deferred MoE finalize requires a fusable RMSNorm flavour"
)
if post_residual_addition is not None:
residual = residual + post_residual_addition
assert self.fusion_service is not None
return self.fusion_service.finalize(
handoff=owed, residual=residual, gamma=gamma
)
def _should_use_all_reduce_rms_norm(
self,
forward_batch: ForwardBatch,
m: int,
residual: Optional[torch.Tensor],
) -> bool:
communicate_fn = self._communicate_with_all_reduce_and_layer_norm_fn
if isinstance(communicate_fn, functools.partial):
norm_fn = communicate_fn.func
residual_input_mode = communicate_fn.keywords.get("residual_input_mode")
else:
norm_fn = communicate_fn
residual_input_mode = None
def _reduce_output_and_update_and_read_residual_cutedsl(
self, owed, residual, forward_batch, post_residual_addition
):
"""Complete the all-reduce the previous layer left with the residual add
and input norm."""
if not isinstance(owed, UnreducedOutput) or not (
self._can_consume_post_moe_all_reduce(
forward_batch, int(owed.partial.shape[0])
)
):
return None
if post_residual_addition is not None:
residual = residual + post_residual_addition
assert self.fusion_service is not None
return self.fusion_service.all_reduce_residual_rms_norm(
local_contribution=owed.partial,
residual=residual,
gamma=_fused_norm_gamma(self.input_layernorm),
)
def _select_mlp_input_fusions(self, residual_input_mode):
fusions = super()._select_mlp_input_fusions(residual_input_mode)
parallel = get_parallel()
return (
self._common_eligible(forward_batch, m)
and residual is not None
and _fused_norm_gamma(self.post_attention_layernorm) is not None
and norm_fn
is CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual
and residual_input_mode is ScatterMode.TP_ATTN_FULL
and self._context.attn_dp_size == 1
if (
residual_input_mode is ScatterMode.TP_ATTN_FULL
and parallel.attn_tp_size == parallel.tp_size
and _fused_norm_gamma(self.post_attention_layernorm) is not None
):
return (
self._mlp_input_reduce_output_and_update_and_read_residual_cutedsl,
*fusions,
)
return fusions
def _mlp_input_reduce_output_and_update_and_read_residual_cutedsl(
self, hidden_states, residual, forward_batch
):
"""The attention output's all-reduce with the residual add and the
post-attention norm."""
if not (
self._common_eligible(forward_batch, int(hidden_states.shape[0]))
and residual is not None
and not get_exec().comm.enable_quant_communications
):
return None
assert self.fusion_service is not None
return self.fusion_service.all_reduce_residual_rms_norm(
local_contribution=hidden_states,
residual=residual,
gamma=_fused_norm_gamma(self.post_attention_layernorm),
)
def _should_use_finalize(self, forward_batch: ForwardBatch, m: int) -> bool:
+21 -6
View File
@@ -23,7 +23,7 @@ from __future__ import annotations
import logging
from contextlib import contextmanager, nullcontext
from functools import cached_property
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
import torch
import torch.nn.functional as F
@@ -2616,9 +2616,24 @@ class DeepseekV2DecoderLayer(nn.Module):
self._gfx95_quant_format = self._detect_gfx95_quant_format()
self.layer_communicator = self._build_layer_communicator(
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
qkv_latent_func=self.self_attn.prepare_qkv_latent,
)
def _build_layer_communicator(
self,
*,
input_layernorm: nn.Module,
post_attention_layernorm: nn.Module,
qkv_latent_func: Optional[Callable],
):
"""The communicator for this layer's norms; it chooses its boundary
steps from them at construction."""
if get_parallel().enable_prefill_cp:
communicator_cls = DSACPLayerCommunicator
elif not is_nextn and _use_mnnvl_cutedsl_fusion():
elif not self.is_nextn and _use_mnnvl_cutedsl_fusion():
# Dense layers too: selecting cutedsl turns the legacy fusion off.
from sglang.srt.layers.moe.cutedsl_ar_fusion import (
CuteDSLFusionLayerCommunicator,
@@ -2627,12 +2642,12 @@ class DeepseekV2DecoderLayer(nn.Module):
communicator_cls = CuteDSLFusionLayerCommunicator
else:
communicator_cls = LayerCommunicator
self.layer_communicator = communicator_cls(
return communicator_cls(
layer_scatter_modes=self.layer_scatter_modes,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
input_layernorm=input_layernorm,
post_attention_layernorm=post_attention_layernorm,
allow_reduce_scatter=True,
qkv_latent_func=self.self_attn.prepare_qkv_latent,
qkv_latent_func=qkv_latent_func,
)
def _detect_gfx95_quant_format(self) -> str:
+6 -2
View File
@@ -438,8 +438,12 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
pre_layernorm=self.post_attention_layernorm if self._use_pre else None,
post_layernorm=self.post_self_attn_layernorm,
)
self.layer_communicator.input_layernorm = attn_prepare_layernorm
self.layer_communicator.post_attention_layernorm = mlp_prepare_layernorm
# The communicator chooses its fused steps from its norms when built.
self.layer_communicator = self._build_layer_communicator(
input_layernorm=attn_prepare_layernorm,
post_attention_layernorm=mlp_prepare_layernorm,
qkv_latent_func=self.layer_communicator.qkv_latent_func,
)
def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool:
if is_nextn: