[Refactor] Leave the FFN reduction to the next layer under attention DP (#41197)

This commit is contained in:
Cheng Wan
2026-09-25 01:47:03 -07:00
committed by GitHub
parent 9d7f44bdbb
commit 963e9fb42d
7 changed files with 570 additions and 47 deletions
+184 -38
View File
@@ -22,6 +22,7 @@ import msgspec
import torch
from sglang.srt.distributed import (
GroupCoordinator,
attention_tensor_model_parallel_all_reduce,
attention_tensor_model_parallel_quant_all_reduce,
tensor_model_parallel_all_reduce,
@@ -626,17 +627,22 @@ def _attn_input_update_and_read_residual(quant_format: str):
class UnreducedOutput(msgspec.Struct, frozen=True):
"""A layer output that still owes its FFN all-reduce, left for the next
layer's input norm. Hand it to the next layer, or pass it through
reduce_output() before reading it any other way."""
layer's input. Hand it to the next layer, or pass it through reduce_output()
before reading it any other way."""
partial: torch.Tensor
# Under attention DP: the reduce-scatter that sums ``partial`` over TP while
# bringing it back to this rank's tokens, in place of the all-reduce.
reduce_and_redistribute: Optional[Callable[[torch.Tensor], torch.Tensor]] = None
def reduce_output(
hidden_states: Union[torch.Tensor, UnreducedOutput, None],
) -> Optional[torch.Tensor]:
"""Run the all-reduce an UnreducedOutput still owes; pass anything else through."""
"""Run the reduction an UnreducedOutput still owes; pass anything else through."""
if isinstance(hidden_states, UnreducedOutput):
if hidden_states.reduce_and_redistribute is not None:
return hidden_states.reduce_and_redistribute(hidden_states.partial)
return deferred_post_experts_all_reduce(hidden_states.partial)
return hidden_states
@@ -667,16 +673,21 @@ def _deferred_reduction_runs_on_the_tp_group() -> bool:
return group is parallel.tp_group
def _ffn_has_tokens(forward_batch: ForwardBatch) -> bool:
if is_dp_attention_enabled():
# The FFN runs on every DP rank's tokens, so every rank decides alike.
return (getattr(forward_batch, "global_dp_buffer_len", None) or 0) > 0
return _batch_size(forward_batch) > 0
def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool:
"""Whether deferred_post_experts_all_reduce is the all-reduce the FFN would
have run itself: one full-precision sum over the same TP group, on an output
that is a plain partial sum, with no attention DP layout to restore. The
output is not a plain partial sum when the MoE combine already summed it, a
replicated shared expert is added after the reduction, or LoRA-B runs on
the unreduced activations."""
that is a plain partial sum. The output is not a plain partial sum when the
MoE combine already summed it, a replicated shared expert is added after the
reduction, or LoRA-B runs on the unreduced activations."""
return (
_batch_size(forward_batch) > 0
and not is_dp_attention_enabled()
_ffn_has_tokens(forward_batch)
and get_moe_a2a_backend().is_none()
and not post_experts_output_is_complete(is_tp_path=True)
and not get_exec().comm.enable_quant_communications
@@ -833,6 +844,13 @@ class LayerCommunicator:
quant_format: str = "",
post_residual_addition: Optional[torch.Tensor] = None,
):
if (
isinstance(hidden_states, UnreducedOutput)
and hidden_states.reduce_and_redistribute is not None
):
# No fused kernel runs under attention DP: the reduce-scatter back to
# this rank's tokens comes first.
hidden_states = reduce_output(hidden_states)
pending = isinstance(hidden_states, UnreducedOutput)
if pending:
hidden_states = hidden_states.partial
@@ -994,6 +1012,28 @@ class LayerCommunicator:
is_layer_sparse=self.layer_scatter_modes.is_layer_sparse,
)
def _reduce_scatter_for_next_layer(
self, forward_batch: ForwardBatch
) -> Optional[Callable[[torch.Tensor], torch.Tensor]]:
"""The reduce-scatter back to this rank's tokens that postprocess would
run on this layer's FFN output, bound for the next layer's input to run
instead; None when postprocess runs anything else."""
if (
self.is_last_layer
or (self._sp_variant is not None and get_forward().sp_active)
or self._communicate_summable_tensor_pair_fn
is not CommunicateSummableTensorPairFn._scatter_hidden_states
):
return None
step = _output_to_local_tokens_step(
forward_batch,
allow_reduce_scatter=self.allow_reduce_scatter,
is_layer_sparse=self.layer_scatter_modes.is_layer_sparse,
)
if step is _redistribute_output:
return None
return partial(_to_local_tokens, step, forward_batch)
def ffn_exit(self, forward_batch: ForwardBatch) -> "FfnExit":
"""Decide once how this layer's FFN output reduction completes. Use the
result as a context manager around the FFN call, then call ``finish``."""
@@ -1111,8 +1151,41 @@ class LayerCommunicator:
and self._ffn_sum_can_move_to_next_layer()
and _unfused_completion_matches_the_ffn(forward_batch)
and not self.should_use_reduce_scatter(forward_batch)
and self._next_layer_input_can_restore_the_layout(forward_batch)
)
def _next_layer_input_can_restore_the_layout(
self, forward_batch: ForwardBatch
) -> bool:
"""Whether the next layer's input can also run what postprocess would
after the FFN's own all-reduce: under attention DP, the scatter back to
this rank's tokens."""
if not is_dp_attention_enabled():
return True
return (
self._communicate_summable_tensor_pair_fn
is CommunicateSummableTensorPairFn._scatter_hidden_states
and not (self._sp_variant is not None and get_forward().sp_active)
and _output_to_local_tokens_step(
forward_batch,
allow_reduce_scatter=self.allow_reduce_scatter,
is_layer_sparse=self.layer_scatter_modes.is_layer_sparse,
)
is _redistribute_output
)
def _scatter_for_next_layer(
self, forward_batch: ForwardBatch
) -> Optional[Callable[[torch.Tensor], torch.Tensor]]:
"""Under attention DP, the all-reduce and the scatter back to this rank's
tokens that the next layer's input runs on a sum this layer's FFN left."""
if (
self._communicate_summable_tensor_pair_fn
is not CommunicateSummableTensorPairFn._scatter_hidden_states
):
return None
return partial(_all_reduce_then_to_local_tokens, forward_batch)
# MOE_FULL gathers across the MoE-CP group, which spans every CP rank when CP is on.
_SCATTER_MODE_SHARDED_AXES = {
@@ -1181,13 +1254,31 @@ class FfnExit:
def finish(
self, hidden_states: torch.Tensor, residual: torch.Tensor
) -> Tuple[Union[torch.Tensor, UnreducedOutput], torch.Tensor]:
"""Leave the reduction to the next layer's input norm, or postprocess."""
"""Leave the reduction to the next layer's input, or postprocess."""
if not isinstance(hidden_states, torch.Tensor):
# A deferred MoE finalize handoff, consumed by the next prepare_attn.
assert self.defer_moe_finalize, "unrequested deferred MoE handoff"
return hidden_states, residual
if self.fuse_mlp_allreduce:
return UnreducedOutput(hidden_states), residual
return (
UnreducedOutput(
hidden_states,
reduce_and_redistribute=self.communicator._scatter_for_next_layer(
self.forward_batch
),
),
residual,
)
reduce_and_redistribute = self.communicator._reduce_scatter_for_next_layer(
self.forward_batch
)
if reduce_and_redistribute is not None:
return (
UnreducedOutput(
hidden_states, reduce_and_redistribute=reduce_and_redistribute
),
residual,
)
return self.communicator.postprocess_layer(
hidden_states, residual, self.forward_batch
)
@@ -1614,6 +1705,81 @@ class CommunicateWithAllReduceAndLayerNormFn:
return hidden_states, residual
def _dp_scatter_group() -> GroupCoordinator:
parallel = get_parallel()
if parallel.tp_size == parallel.attn_dp_size:
return parallel.tp_group
return parallel.attn_tp_group
def _reduce_and_redistribute_output_varlen(
local_hidden_states: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
) -> None:
get_parallel().tp_group.reduce_scatterv(
hidden_states,
output=local_hidden_states,
sizes=get_dp_global_num_tokens(),
)
def _reduce_and_redistribute_output_max_len(
local_hidden_states: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
) -> None:
dp_reduce_scatter_tensor(local_hidden_states, hidden_states)
def _redistribute_output(
local_hidden_states: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
) -> None:
dp_scatter(local_hidden_states, hidden_states, forward_batch)
def _output_to_local_tokens_step(
forward_batch: ForwardBatch, *, allow_reduce_scatter: bool, is_layer_sparse: bool
) -> Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]:
"""How a FULL-layout layer output comes back to this rank's tokens under
attention DP: a reduce-scatter when the FFN left its sum to it, otherwise a
scatter of the reduced output."""
# A MoE block leaves its sum to reduce_scatterv whenever it applies
# (should_skip_post_experts_all_reduce); a dense MLP does only under the
# published mlp_reduce_scatter, which needs allow_reduce_scatter.
if should_use_dp_reduce_scatterv() and (allow_reduce_scatter or is_layer_sparse):
return _reduce_and_redistribute_output_varlen
if (
allow_reduce_scatter
and forward_batch.dp_padding_mode.is_max_len()
and can_use_dp_reduce_scatter()
):
return _reduce_and_redistribute_output_max_len
return _redistribute_output
def _to_local_tokens(
step: Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None],
forward_batch: ForwardBatch,
hidden_states: torch.Tensor,
) -> torch.Tensor:
local_hidden_states = get_local_dp_buffer(_dp_scatter_group())
step(local_hidden_states, hidden_states, forward_batch)
return local_hidden_states
def _all_reduce_then_to_local_tokens(
forward_batch: ForwardBatch, hidden_states: torch.Tensor
) -> torch.Tensor:
return _to_local_tokens(
_redistribute_output,
forward_batch,
deferred_post_experts_all_reduce(hidden_states),
)
class CommunicateSummableTensorPairFn:
"""It is allowed to make (hidden_states, residual) := (hidden_states + residual, None) if needed."""
@@ -1693,34 +1859,14 @@ class CommunicateSummableTensorPairFn:
allow_reduce_scatter: bool = False,
is_layer_sparse: bool = False,
):
if get_parallel().tp_size == get_parallel().attn_dp_size:
group = get_parallel().tp_group
else:
group = get_parallel().attn_tp_group
hidden_states, global_hidden_states = (
get_local_dp_buffer(group),
hidden_states,
local_hidden_states = get_local_dp_buffer(_dp_scatter_group())
step = _output_to_local_tokens_step(
forward_batch,
allow_reduce_scatter=allow_reduce_scatter,
is_layer_sparse=is_layer_sparse,
)
# A MoE block leaves its sum to reduce_scatterv whenever it applies
# (should_skip_post_experts_all_reduce); a dense MLP does only under the
# published mlp_reduce_scatter, which needs allow_reduce_scatter.
if should_use_dp_reduce_scatterv() and (
allow_reduce_scatter or is_layer_sparse
):
get_parallel().tp_group.reduce_scatterv(
global_hidden_states,
output=hidden_states,
sizes=get_dp_global_num_tokens(),
)
elif (
allow_reduce_scatter
and forward_batch.dp_padding_mode.is_max_len()
and can_use_dp_reduce_scatter()
):
dp_reduce_scatter_tensor(hidden_states, global_hidden_states)
else:
dp_scatter(hidden_states, global_hidden_states, forward_batch)
return hidden_states, residual
step(local_hidden_states, hidden_states, forward_batch)
return local_hidden_states, residual
@staticmethod
def _gather(