[Refactor] Choose a GQA prefill CP extend's steps from declarations (#41421)

This commit is contained in:
Cheng Wan
2026-09-27 16:48:17 -07:00
committed by GitHub
parent 380a4d0332
commit 7a833ad0a2
12 changed files with 268 additions and 90 deletions
+135 -67
View File
@@ -16,7 +16,7 @@ from contextlib import contextmanager
from dataclasses import dataclass
from enum import Enum, auto
from functools import cached_property, partial
from typing import Callable, Dict, Optional, Tuple, Union
from typing import Callable, Dict, List, Optional, Tuple, Union
import msgspec
import torch
@@ -767,6 +767,17 @@ class LayerCommunicator:
if sides is not None
else self._select_boundaries_from_scatter_modes()
)
# The steps a batch that shards its tokens over attention CP runs; None
# without CP or where the steps come from the scatter modes.
self._cp_steps = (
_select_boundary_steps(
self._declared_sides(cp_active=True),
fusions=self._select_mlp_input_fusions(),
force_layernorm_before_gather=force_layernorm_before_dp_gather,
)
if sides is not None and get_parallel().attn_cp_size > 1
else None
)
# The steps a batch with input-scattered attention runs; None where it
# cannot run or the steps come from the scatter modes.
self._input_scattered_steps = (
@@ -815,11 +826,16 @@ class LayerCommunicator:
and get_moe_a2a_backend().is_none()
)
def _declared_sides(self) -> Optional[DecoderLayerSides]:
def _declared_sides(
self, *, cp_active: bool = False
) -> Optional[DecoderLayerSides]:
"""The declarations this layer's boundaries are chosen from: an
attention and an FFN, without CP, in a layer whose previous layer is
known. None when the steps come from the scatter modes. An active
LayerNorm SP region and input-scattered attention have their own."""
attention and an FFN, in a layer whose previous layer is known, with
attention CP only for a prefill CP whose FFN gathers over the whole CP
group. None when the steps come from the scatter modes. ``cp_active``
gives a batch that shards its tokens over CP; the others hold every
token on each CP rank. An active LayerNorm SP region and input-scattered
attention have their own."""
modes = self.layer_scatter_modes
parallel = get_parallel()
# A MoE dispatched per DP shard computes on this rank's local rows and
@@ -827,13 +843,16 @@ class LayerCommunicator:
moe_on_local_rows = is_moe_input_scattered_across_dp_ranks()
if not (
self._takes_declared_boundaries
and parallel.attn_cp_size == 1
and (parallel.attn_cp_size == 1 or _cp_on_declarations())
and not enable_moe_dense_fully_dp()
and (modes.is_first_layer or modes.is_previous_layer_sparse is not None)
):
return None
# A sum over rows gathered across CP stays with this layer: the next
# layer's input holds only this rank's chunk.
may_leave = not cp_active
return decoder_layer_sides(
axis_sizes=_token_axis_sizes(),
axis_sizes=_token_axis_sizes(cp_active=cp_active),
ffn_on_local_rows=modes.is_layer_sparse and moe_on_local_rows,
previous_on_local_rows=(
not modes.is_first_layer
@@ -843,13 +862,14 @@ class LayerCommunicator:
is_last_layer=modes.is_last_layer,
attention_gathers_local_rows=_use_ag_after_qlora,
ffn_group=SumGroup.MOE_OUTPUT if modes.is_layer_sparse else SumGroup.TP,
leaves_for_next_layer=self.allow_deferred_ffn_reduction,
leaves_for_reduce_scatter=self.allow_reduce_scatter,
leaves_for_next_layer=self.allow_deferred_ffn_reduction and may_leave,
leaves_for_reduce_scatter=self.allow_reduce_scatter and may_leave,
# A MoE block leaves its sum to reduce_scatterv whenever that combine
# applies (should_skip_post_experts_all_reduce).
leaves_for_reduce_scatterv=(
self.allow_reduce_scatter or modes.is_layer_sparse
),
)
and may_leave,
)
def _select_boundaries_from_scatter_modes(self) -> "BoundarySteps":
@@ -924,7 +944,9 @@ class LayerCommunicator:
post_residual_addition=post_residual_addition,
)
if captured_last_layer_outputs is not None:
gathered_last_layer_output = self._batch_steps().attention_input(
gathered_last_layer_output = self._batch_steps(
forward_batch
).attention_input(
hidden_states=residual,
forward_batch=forward_batch,
context=self._context,
@@ -933,13 +955,15 @@ class LayerCommunicator:
gathered_last_layer_output is residual
# An accumulator that copies on append already holds a snapshot.
and not getattr(captured_last_layer_outputs, "copies_on_append", False)
and not self._post_attn_residual_is_read_only(residual)
and not self._post_attn_residual_is_read_only(residual, forward_batch)
):
gathered_last_layer_output = residual.clone()
captured_last_layer_outputs.append(gathered_last_layer_output)
return hidden_states, residual
def _post_attn_residual_is_read_only(self, residual: torch.Tensor) -> bool:
def _post_attn_residual_is_read_only(
self, residual: torch.Tensor, forward_batch: ForwardBatch
) -> bool:
"""True if ``prepare_mlp``'s post-attention RMSNorm leaves ``residual``
untouched, so Eagle3 aux capture can keep its reference and skip the clone.
@@ -950,7 +974,10 @@ class LayerCommunicator:
the batch is not input-scattered.
"""
return (
any(f.may_return_new_residual for f in self._batch_steps().fused)
any(
f.may_return_new_residual
for f in self._batch_steps(forward_batch).fused
)
and not get_attn_tp_context().input_scattered
and apply_flashinfer_allreduce_fusion(residual.shape[0])
)
@@ -1012,7 +1039,7 @@ class LayerCommunicator:
)
if get_forward().sp_active:
hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
layer_input = self._batch_steps().layer_input
layer_input = self._batch_steps(forward_batch).layer_input
if layer_input is not None:
hidden_states, residual = layer_input(
hidden_states, residual, self._context
@@ -1038,9 +1065,10 @@ class LayerCommunicator:
def _in_sp_region(self) -> bool:
return self._sp_steps is not None and get_forward().sp_active
def _batch_steps(self) -> "BoundarySteps":
def _batch_steps(self, forward_batch: ForwardBatch) -> "BoundarySteps":
"""The steps this batch runs: an active LayerNorm SP region's,
input-scattered attention's, or the layer's ordinary ones."""
input-scattered attention's, a CP extend's, or the layer's ordinary
ones."""
if self._in_sp_region():
return self._sp_steps
if (
@@ -1048,11 +1076,13 @@ class LayerCommunicator:
and get_attn_tp_context().input_scattered
):
return self._input_scattered_steps
if self._cp_steps is not None and moe_cp_gathered_rows(forward_batch):
return self._cp_steps
return self._steps
def _finish_prepare_attn(self, hidden_states, residual, forward_batch):
"""Tail every prepare_attn path must run, or ``attn_inputs`` is unset."""
hidden_states = self._batch_steps().attention_input(
hidden_states = self._batch_steps(forward_batch).attention_input(
hidden_states=hidden_states,
forward_batch=forward_batch,
context=self._context,
@@ -1195,7 +1225,7 @@ class LayerCommunicator:
if cache is not None:
self._context.cache = cache
return self._batch_steps().ffn_input(
return self._batch_steps(forward_batch).ffn_input(
hidden_states,
residual,
forward_batch,
@@ -1216,7 +1246,7 @@ class LayerCommunicator:
residual: torch.Tensor,
forward_batch: ForwardBatch,
):
steps = self._batch_steps()
steps = self._batch_steps(forward_batch)
if steps.returns_over_dp:
step = self._postprocess_dp_step(forward_batch) or _redistribute_output
return _to_local_tokens(step, forward_batch, hidden_states), residual
@@ -1229,10 +1259,12 @@ class LayerCommunicator:
is_layer_sparse=self.layer_scatter_modes.is_layer_sparse,
)
def _local_token_move_can_go_to_next_layer(self) -> bool:
def _local_token_move_can_go_to_next_layer(
self, forward_batch: ForwardBatch
) -> bool:
"""Whether the next layer's input can run this layer's move of its FFN
output back to this rank's tokens: the base postprocess scatter."""
return self._batch_steps().returns_over_dp
return self._batch_steps(forward_batch).returns_over_dp
def _postprocess_dp_step(
self, forward_batch: ForwardBatch
@@ -1240,7 +1272,7 @@ class LayerCommunicator:
"""The reduce-scatter that brings this layer's FFN output back to this
rank's tokens under attention DP; None when the base postprocess would
only scatter, or does not move tokens."""
steps = self._batch_steps()
steps = self._batch_steps(forward_batch)
if not steps.returns_over_dp:
return None
return _reduce_and_redistribute_output_step(
@@ -1255,7 +1287,7 @@ class LayerCommunicator:
"""Whether the FFN leaves its sum out because a reduce-scatter completes
it: the attention-DP one ``dp_step`` names, or the CP / input-scattered
one."""
if not self._batch_steps().ffn_output.leaves_for_reduce_scatter:
if not self._batch_steps(forward_batch).ffn_output.leaves_for_reduce_scatter:
return False
if dp_step is not None:
return True
@@ -1266,10 +1298,10 @@ class LayerCommunicator:
return True
return get_attn_tp_context().input_scattered and not self.is_last_layer
def ffn_reduction_group(self) -> GroupCoordinator:
def ffn_reduction_group(self, forward_batch: ForwardBatch) -> GroupCoordinator:
"""The group this layer's FFN output owes its sum over: the MoE output's
group on a sparse layer, the TP group a dense MLP reduces over."""
return _sum_group(self._batch_steps().ffn_output.group)
return _sum_group(self._batch_steps(forward_batch).ffn_output.group)
def _select_ffn_completion(self, forward_batch: ForwardBatch) -> "FfnCompletion":
"""Decide once, before the FFN runs, what it skips and what completes its
@@ -1283,7 +1315,7 @@ class LayerCommunicator:
forward_batch=forward_batch,
dp_step=dp_step,
)
steps = self._batch_steps()
steps = self._batch_steps(forward_batch)
if not steps.ffn_output.leaves_for_next_layer:
return FfnCompletion(
defer_moe_finalize=False,
@@ -1297,7 +1329,7 @@ class LayerCommunicator:
forward_batch, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step
)
if fuse_mlp_allreduce:
group = self.ffn_reduction_group()
group = self.ffn_reduction_group(forward_batch)
if steps.returns_over_dp:
# Under attention DP the next layer also brings the sum back to
# this rank's tokens.
@@ -1313,7 +1345,7 @@ class LayerCommunicator:
elif (
dp_step is not None
and not self.is_last_layer
and self._local_token_move_can_go_to_next_layer()
and self._local_token_move_can_go_to_next_layer(forward_batch)
):
complete = partial(
_leave_to_next_layer,
@@ -1343,7 +1375,7 @@ class LayerCommunicator:
) -> Tuple[torch.Tensor, torch.Tensor]:
"""This layer's postprocess, run with the attention-DP step the FFN exit
chose."""
if self._local_token_move_can_go_to_next_layer():
if self._local_token_move_can_go_to_next_layer(forward_batch):
return (
_to_local_tokens(
dp_step or _redistribute_output, forward_batch, hidden_states
@@ -1381,10 +1413,13 @@ class LayerCommunicator:
"""Whether the MoE may hand an unfinalized output to the next layer."""
return False
def _ffn_sum_can_move_to_next_layer(self) -> bool:
def _ffn_sum_can_move_to_next_layer(self, forward_batch: ForwardBatch) -> bool:
# Under the MoE-CP all-gather the fusion path would skip postprocess_layer
# and its MoE-CP scatter, leaving hidden_states longer than the residual.
if is_enable_moe_cp_allgather() or not self._batch_steps().ffn_sum_is_movable:
if (
is_enable_moe_cp_allgather()
or not self._batch_steps(forward_batch).ffn_sum_is_movable
):
return False
# The fused residual+LN reduces over a single group. Hybrid EP+TP spans
@@ -1412,7 +1447,7 @@ class LayerCommunicator:
def should_fuse_mlp_allreduce_with_next_layer(
self, forward_batch: ForwardBatch
) -> bool:
if not self._ffn_sum_can_move_to_next_layer():
if not self._ffn_sum_can_move_to_next_layer(forward_batch):
return False
batch_size = _batch_size(forward_batch)
return (
@@ -1446,14 +1481,17 @@ class LayerCommunicator:
return (
self._context.tp_size > 1
and not self.is_last_layer
and self._ffn_sum_can_move_to_next_layer()
and self._ffn_sum_can_move_to_next_layer(forward_batch)
and _unfused_completion_matches_the_ffn(forward_batch)
and not mlp_reduce_scatter
# Under attention DP the next layer must also run postprocess's
# scatter back to this rank's tokens, and nothing more.
and (
not is_dp_attention_enabled()
or (self._local_token_move_can_go_to_next_layer() and dp_step is None)
or (
self._local_token_move_can_go_to_next_layer(forward_batch)
and dp_step is None
)
)
)
@@ -1689,14 +1727,28 @@ def _reduce_and_redistribute_output_to_attn_tp_shards(
return local_hidden_states
def moe_cp_gathered_rows(forward_batch: ForwardBatch) -> Optional[List[int]]:
"""The real rows each MoE-CP rank contributes when this batch's FFN input is
gathered across the MoE-CP group, or None when it is not: only a context
parallel extend with CP metadata gathers, and only when the group has more
than one rank. The batch is read first, so a batch that is not a CP extend
never reads the group."""
if (
forward_batch.forward_mode.is_context_parallel_extend()
and forward_batch.attn_cp_metadata is not None
and get_moe_cp_size() > 1
):
return forward_batch.attn_cp_metadata.per_rank_actual_token
return None
def _redistribute_input_to_moe_cp(
hidden_states: torch.Tensor, forward_batch: ForwardBatch, moe_cp_size: int
hidden_states: torch.Tensor, rows: List[int], moe_cp_size: int
) -> torch.Tensor:
# Zigzag split can produce unequal token counts across CP ranks
# (when seq_len % (cp_size * 2) != 0). NCCL allgather requires
# equal input sizes, so pad to the max per-rank token count.
per_rank_tokens = forward_batch.attn_cp_metadata.per_rank_actual_token
max_tokens = max(per_rank_tokens)
max_tokens = max(rows)
pad_size = max_tokens - hidden_states.shape[0]
if pad_size > 0:
hidden_states = torch.nn.functional.pad(hidden_states, [0, 0, 0, pad_size])
@@ -1853,15 +1905,30 @@ def _sum_group(group: SumGroup) -> GroupCoordinator:
return post_experts_reduction_group()
def _token_axis_sizes() -> Dict[TokenAxis, int]:
def _token_axis_sizes(*, cp_active: bool = False) -> Dict[TokenAxis, int]:
"""The token axes' sizes for a batch: attention CP shards tokens only on
a CP extend (``cp_active``); otherwise every CP rank holds them all."""
parallel = get_parallel()
return {
TokenAxis.ATTN_DP: parallel.attn_dp_size,
TokenAxis.ATTN_CP: parallel.attn_cp_size,
TokenAxis.ATTN_CP: parallel.attn_cp_size if cp_active else 1,
TokenAxis.ATTN_TP_SCATTER: parallel.attn_tp_size,
}
def _cp_on_declarations() -> bool:
"""Whether attention CP is one the declarations cover: a prefill CP that
shards tokens, without attention DP, whose FFN input gathers over a MoE-CP
group that is the whole CP group. DSA and MLA CP pick their own steps."""
parallel = get_parallel()
return (
_generic_prefill_cp_shards_tokens()
and not (is_dsa_enable_prefill_cp() or is_mla_cp_enabled())
and parallel.attn_dp_size == 1
and parallel.moe_dp_size == 1
)
class BoundarySteps(msgspec.Struct, frozen=True):
"""The steps a batch runs at a layer's boundaries: into the attention,
from the attention output to the FFN input, and the FFN output on to the
@@ -1999,6 +2066,19 @@ def _select_ffn_input(
partial(_mlp_input_scatter, scatters_residual=residual != residual_to),
(),
)
if gathered == {TokenAxis.ATTN_CP}:
# Each CP rank completes its own chunk, then the chunks are gathered
# over the MoE-CP group, each padded to the longest.
on_chunk, fused = _select_ffn_input(
produced,
residual=residual,
residual_to=residual_to,
need=StageInput(produced.layout),
force_layernorm_before_gather=force_layernorm_before_gather,
fusions=fusions,
residual_joins_sum=residual_joins_sum,
)
return partial(_mlp_input_gather_moe_cp, gather=on_chunk), fused
if (
residual_to != produced.layout
or gathered not in (frozenset(), {TokenAxis.ATTN_DP})
@@ -2055,11 +2135,12 @@ def _select_ffn_output_move(
return False, CommunicateSummableTensorPairFn._gather
raise NotImplementedError(f"{produced=} {residual=} {to=}")
returned = residual.sharded - produced.layout.sharded
if (
to != residual
or not produced.layout.sharded <= residual.sharded
or returned != {TokenAxis.ATTN_DP}
):
if to != residual or not produced.layout.sharded <= residual.sharded:
raise NotImplementedError(f"{produced=} {residual=} {to=}")
if returned == {TokenAxis.ATTN_CP}:
# This rank's chunk of the rows gathered over CP; no collective.
return False, CommunicateSummableTensorPairFn._scatter_hidden_states_moe
if returned != {TokenAxis.ATTN_DP}:
raise NotImplementedError(f"{produced=} {residual=} {to=}")
return True, None
@@ -2240,16 +2321,10 @@ def _mlp_input_gather_moe_cp(
hidden_states, residual, forward_batch, layernorm, context
)
# Only active during prefill (context-parallel extend); decode keeps existing path.
moe_cp_size = get_moe_cp_size()
if (
moe_cp_size > 1
and hidden_states.shape[0] > 0
and forward_batch.forward_mode.is_context_parallel_extend()
and forward_batch.attn_cp_metadata is not None
):
rows = moe_cp_gathered_rows(forward_batch)
if rows is not None and hidden_states.shape[0] > 0:
hidden_states = _redistribute_input_to_moe_cp(
hidden_states, forward_batch, moe_cp_size
hidden_states, rows, get_moe_cp_size()
)
return hidden_states, residual
@@ -2291,14 +2366,13 @@ def _redistribute_output(
def _redistribute_output_from_moe_cp(
hidden_states: torch.Tensor, forward_batch: ForwardBatch
hidden_states: torch.Tensor, rows: List[int]
) -> torch.Tensor:
moe_cp_rank = get_moe_cp_rank()
# The allgather was padded to max_tokens_per_rank (equal chunks).
# Extract this rank's actual (non-padded) tokens from its chunk.
per_rank_tokens = forward_batch.attn_cp_metadata.per_rank_actual_token
max_tokens_per_rank = max(per_rank_tokens)
actual_local_tokens = per_rank_tokens[moe_cp_rank]
max_tokens_per_rank = max(rows)
actual_local_tokens = rows[moe_cp_rank]
return hidden_states.narrow(
0, moe_cp_rank * max_tokens_per_rank, actual_local_tokens
).contiguous()
@@ -2459,15 +2533,9 @@ class CommunicateSummableTensorPairFn:
# Only scatter back during prefill; decode was never allgathered so no-op.
# Safe w.r.t. empty tensors: same reasoning as _mlp_input_gather_moe_cp
# — CP extend always has non-zero tokens per rank, and decode skips this path.
moe_cp_size = get_moe_cp_size()
if (
moe_cp_size > 1
and forward_batch.forward_mode.is_context_parallel_extend()
and forward_batch.attn_cp_metadata is not None
):
hidden_states = _redistribute_output_from_moe_cp(
hidden_states, forward_batch
)
rows = moe_cp_gathered_rows(forward_batch)
if rows is not None:
hidden_states = _redistribute_output_from_moe_cp(hidden_states, rows)
if context.attn_dp_size > 1:
hidden_states = _to_local_tokens(
@@ -1230,7 +1230,11 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
def moe_num_token_non_padded(self) -> Optional[torch.Tensor]:
"""Bound for masking a sparse MoE's padded rows, or None when the MoE
input is a gathered buffer whose real rows are not a prefix of it."""
from sglang.srt.layers.communicator import ScatterMode, sparse_mlp_scatter_mode
from sglang.srt.layers.communicator import (
ScatterMode,
moe_cp_gathered_rows,
sparse_mlp_scatter_mode,
)
if self.num_token_non_padded is None:
return None
@@ -1239,7 +1243,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
if mode == ScatterMode.SCATTERED:
# a2a dispatch, FP4 all-gather and dwdp all route the local shard.
return self.num_token_non_padded
if mode == ScatterMode.MOE_FULL and self._moe_input_gathered_across_moe_cp():
if mode == ScatterMode.MOE_FULL and moe_cp_gathered_rows(self) is not None:
return None
# DSA / MLA CP take the FULL mode but all-gather across CP on a prefill,
# which leaves the real rows zigzag-permuted rather than in a prefix.
@@ -1255,16 +1259,6 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# None under cuda-graph replay: its static batch carries no GLOBAL count.
return self.global_num_token_non_padded
def _moe_input_gathered_across_moe_cp(self) -> bool:
from sglang.srt.layers.dp_attention import get_moe_cp_size
# Mirrors the communicator's MOE_FULL gather guard: decode stays FULL.
return (
self.forward_mode.is_context_parallel_extend()
and self.attn_cp_metadata is not None
and get_moe_cp_size() > 1
)
def _moe_input_gathered_across_cp(self) -> bool:
from sglang.srt.layers.attention.dsa.utils import dsa_use_prefill_cp
from sglang.srt.layers.cp.utils import is_mla_cp_active
+2 -1
View File
@@ -428,7 +428,8 @@ class NemotronHMLPLikeDecoderLayer(nn.Module):
hidden_states = self.mixer.forward(hidden_states)
if fuse_mlp_allreduce:
hidden_states = UnreducedOutput(
hidden_states, group=self.layer_communicator.ffn_reduction_group()
hidden_states,
group=self.layer_communicator.ffn_reduction_group(forward_batch),
)
else:
hidden_states, residual = self.layer_communicator.postprocess_layer(