[Refactor] Decide an FFN exit's completion once and declare the group it owes (#41200)

This commit is contained in:
Cheng Wan
2026-09-25 01:50:35 -07:00
committed by GitHub
parent 26a3214cda
commit 3450d68d4e
12 changed files with 346 additions and 242 deletions
+136 -107
View File
@@ -66,9 +66,9 @@ from sglang.srt.layers.flashinfer_comm_fusion import (
)
from sglang.srt.layers.moe import (
can_merge_post_experts_all_reduce,
deferred_post_experts_all_reduce,
get_moe_a2a_backend,
post_experts_output_is_complete,
post_experts_reduction_group,
should_use_dp_reduce_scatterv,
should_use_flashinfer_cutlass_moe_fp4_allgather,
)
@@ -629,13 +629,15 @@ def _attn_input_update_and_read_residual(quant_format: str):
class UnreducedOutput(msgspec.Struct, frozen=True):
"""A layer output that still owes its FFN all-reduce, left for the next
layer's input. Hand it to the next layer, or pass it through reduce_output()
before reading it any other way."""
"""A layer output that still owes its sum, left for the next layer's input.
Hand it to the next layer, or pass it through reduce_output() before reading
it any other way. The producer says what is owed: one all-reduce over
``group`` that keeps the layout, or ``reduce_and_redistribute``."""
partial: torch.Tensor
# Under attention DP: the reduce-scatter that sums ``partial`` over TP while
# bringing it back to this rank's tokens, in place of the all-reduce.
group: Optional[GroupCoordinator] = None
# Under attention DP: the reduction that also brings ``partial`` back to this
# rank's tokens (a reduce-scatter, or an all-reduce then a scatter).
reduce_and_redistribute: Optional[Callable[[torch.Tensor], torch.Tensor]] = None
@@ -646,7 +648,7 @@ def reduce_output(
if isinstance(hidden_states, UnreducedOutput):
if hidden_states.reduce_and_redistribute is not None:
return hidden_states.reduce_and_redistribute(hidden_states.partial)
return deferred_post_experts_all_reduce(hidden_states.partial)
return hidden_states.group.all_reduce(hidden_states.partial)
return hidden_states
@@ -666,14 +668,13 @@ def _batch_size(forward_batch: ForwardBatch) -> int:
def _deferred_reduction_runs_on_the_tp_group() -> bool:
"""Whether deferred_post_experts_all_reduce reduces over the TP group object
"""Whether an MoE output's one all-reduce runs over the TP group object
itself, the group a dense FFN and reduce_moe_output reduce over."""
parallel = get_parallel()
if parallel.moe_ep_size > 1 and parallel.moe_tp_size > 1:
# Some MoE blocks reduce EP and MoE-TP in two steps instead of merging.
return False
group = parallel.moe_ep_group if parallel.moe_ep_size > 1 else parallel.moe_tp_group
return group is parallel.tp_group
return post_experts_reduction_group() is parallel.tp_group
def _ffn_has_tokens(forward_batch: ForwardBatch) -> bool:
@@ -684,8 +685,8 @@ def _ffn_has_tokens(forward_batch: ForwardBatch) -> bool:
def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool:
"""Whether deferred_post_experts_all_reduce is the all-reduce the FFN would
have run itself: one full-precision sum over the same TP group, on an output
"""Whether the next layer's all-reduce is the one the FFN would have run
itself: one full-precision sum over the same TP group, on an output
that is a plain partial sum. The output is not a plain partial sum when the
MoE combine already summed it, a replicated shared expert is added after the
reduction, or LoRA-B runs on the unreduced activations."""
@@ -729,6 +730,13 @@ class LayerCommunicator:
force_layernorm_before_dp_gather
)
self._post_init_communicate()
# Under attention DP, the base postprocess scatters the FFN output back
# to this rank's tokens, which the next layer's input can run instead;
# the MHC and DSA-CP postprocess do more, so theirs stays here.
self._postprocess_scatters_to_local_tokens = (
self._communicate_summable_tensor_pair_fn
is CommunicateSummableTensorPairFn._scatter_hidden_states
)
self._speculative_algo = SpeculativeAlgorithm.from_string(
get_spec().speculative_algorithm
)
@@ -855,9 +863,12 @@ class LayerCommunicator:
# No fused kernel runs under attention DP: the reduce-scatter back to
# this rank's tokens comes first.
hidden_states = reduce_output(hidden_states)
pending = isinstance(hidden_states, UnreducedOutput)
unreduced = (
hidden_states if isinstance(hidden_states, UnreducedOutput) else None
)
pending = unreduced is not None
if pending:
hidden_states = hidden_states.partial
hidden_states = unreduced.partial
# residual is None marks the first decoder layer, where the SP region
# opens: re-evaluated per forward so a crash mid-loop cannot leak into
# the next one.
@@ -889,12 +900,14 @@ class LayerCommunicator:
hidden_states, residual, forward_batch
)
if post_residual_addition is None
# The fused kernel reduces over the MoE output's group.
and unreduced.group is post_experts_reduction_group()
else None
)
if fused is not None:
hidden_states, residual = fused
else:
hidden_states = deferred_post_experts_all_reduce(hidden_states)
hidden_states = reduce_output(unreduced)
hidden_states, residual = _attn_input_update_and_read_residual(
quant_format
)(self.input_layernorm, hidden_states, residual, post_residual_addition)
@@ -1016,27 +1029,74 @@ class LayerCommunicator:
is_layer_sparse=self.layer_scatter_modes.is_layer_sparse,
)
def _reduce_scatter_for_next_layer(
def _local_token_move_can_go_to_next_layer(self) -> bool:
"""Whether the next layer's input can run this layer's move of its FFN
output back to this rank's tokens: the base postprocess scatter, outside
an active LayerNorm SP region."""
return self._postprocess_scatters_to_local_tokens and not (
self._sp_variant is not None and get_forward().sp_active
)
def _reduce_scatter_step(
self, forward_batch: ForwardBatch
) -> Optional[Callable[[torch.Tensor], torch.Tensor]]:
"""The reduce-scatter back to this rank's tokens that postprocess would
run on this layer's FFN output, bound for the next layer's input to run
instead; None when postprocess runs anything else."""
if (
self.is_last_layer
or (self._sp_variant is not None and get_forward().sp_active)
or self._communicate_summable_tensor_pair_fn
is not CommunicateSummableTensorPairFn._scatter_hidden_states
):
return None
step = _output_to_local_tokens_step(
) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]:
"""The reduce-scatter postprocess would run on this layer's FFN output;
None when it runs only a scatter, or anything else."""
return _reduce_and_redistribute_output_step(
forward_batch,
allow_reduce_scatter=self.allow_reduce_scatter,
is_layer_sparse=self.layer_scatter_modes.is_layer_sparse,
)
if step is _redistribute_output:
return None
return partial(_to_local_tokens, step, forward_batch)
def ffn_reduction_group(self) -> GroupCoordinator:
"""The group this layer's FFN output owes its sum over: the MoE output's
group on a sparse layer, the TP group a dense MLP reduces over."""
if self.layer_scatter_modes.is_layer_sparse:
return post_experts_reduction_group()
return get_parallel().tp_group
def _select_ffn_completion(self, forward_batch: ForwardBatch) -> "FfnCompletion":
"""Decide once, before the FFN runs, what it skips and what completes its
output: the next layer's input, or this layer's postprocess."""
defer_moe_finalize = self.should_defer_moe_finalize(forward_batch)
# Deferring implies fusing: a handoff skips the post-experts all-reduce.
fuse_mlp_allreduce = defer_moe_finalize or self.should_defer_ffn_reduction(
forward_batch
)
mlp_reduce_scatter = self.should_use_reduce_scatter(forward_batch)
if fuse_mlp_allreduce:
group = self.ffn_reduction_group()
if self._postprocess_scatters_to_local_tokens:
# Under attention DP the next layer also brings the sum back to
# this rank's tokens.
leave = partial(
UnreducedOutput,
reduce_and_redistribute=partial(
_all_reduce_then_to_local_tokens, group, forward_batch
),
)
else:
leave = partial(UnreducedOutput, group=group)
elif not self.is_last_layer and self._local_token_move_can_go_to_next_layer():
step = self._reduce_scatter_step(forward_batch)
leave = (
None
if step is None
else partial(
UnreducedOutput,
reduce_and_redistribute=partial(
_to_local_tokens, step, forward_batch
),
)
)
else:
leave = None
return FfnCompletion(
defer_moe_finalize=defer_moe_finalize,
fuse_mlp_allreduce=fuse_mlp_allreduce,
mlp_reduce_scatter=mlp_reduce_scatter,
leave=leave,
)
def ffn_exit(self, forward_batch: ForwardBatch) -> "FfnExit":
"""Decide once how this layer's FFN output reduction completes. Use the
@@ -1155,41 +1215,17 @@ class LayerCommunicator:
and self._ffn_sum_can_move_to_next_layer()
and _unfused_completion_matches_the_ffn(forward_batch)
and not self.should_use_reduce_scatter(forward_batch)
and self._next_layer_input_can_restore_the_layout(forward_batch)
)
def _next_layer_input_can_restore_the_layout(
self, forward_batch: ForwardBatch
) -> bool:
"""Whether the next layer's input can also run what postprocess would
after the FFN's own all-reduce: under attention DP, the scatter back to
this rank's tokens."""
if not is_dp_attention_enabled():
return True
return (
self._communicate_summable_tensor_pair_fn
is CommunicateSummableTensorPairFn._scatter_hidden_states
and not (self._sp_variant is not None and get_forward().sp_active)
and _output_to_local_tokens_step(
forward_batch,
allow_reduce_scatter=self.allow_reduce_scatter,
is_layer_sparse=self.layer_scatter_modes.is_layer_sparse,
# Under attention DP the next layer must also run postprocess's
# scatter back to this rank's tokens, and nothing more.
and (
not is_dp_attention_enabled()
or (
self._local_token_move_can_go_to_next_layer()
and self._reduce_scatter_step(forward_batch) is None
)
)
is _redistribute_output
)
def _scatter_for_next_layer(
self, forward_batch: ForwardBatch
) -> Optional[Callable[[torch.Tensor], torch.Tensor]]:
"""Under attention DP, the all-reduce and the scatter back to this rank's
tokens that the next layer's input runs on a sum this layer's FFN left."""
if (
self._communicate_summable_tensor_pair_fn
is not CommunicateSummableTensorPairFn._scatter_hidden_states
):
return None
return partial(_all_reduce_then_to_local_tokens, forward_batch)
# MOE_FULL gathers across the MoE-CP group, which spans every CP rank when CP is on.
_SCATTER_MODE_SHARDED_AXES = {
@@ -1218,10 +1254,21 @@ def scatter_mode_layouts(
}
class FfnCompletion(msgspec.Struct, frozen=True):
"""One FFN's reduction decision. The flags are published while the FFN runs;
``leave`` wraps its output for the next layer's input to complete, or is None
when this layer's postprocess completes it."""
defer_moe_finalize: bool
fuse_mlp_allreduce: bool
mlp_reduce_scatter: bool
leave: Optional[Callable[[torch.Tensor], UnreducedOutput]] = None
class FfnExit:
"""One FFN's reduction decision. Inside the ``with`` block it is published as
``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` / ``defer_moe_finalize`` on
``get_forward()``."""
"""The scope that publishes an FfnCompletion while the FFN runs: inside the
``with`` block it is ``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` /
``defer_moe_finalize`` on ``get_forward()``."""
__slots__ = (
"communicator",
@@ -1229,19 +1276,18 @@ class FfnExit:
"defer_moe_finalize",
"fuse_mlp_allreduce",
"mlp_reduce_scatter",
"_leave",
"_scope",
)
def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch):
self.communicator = communicator
self.forward_batch = forward_batch
self.defer_moe_finalize = communicator.should_defer_moe_finalize(forward_batch)
# Deferring implies fusing: a handoff skips the post-experts all-reduce.
self.fuse_mlp_allreduce = (
self.defer_moe_finalize
or communicator.should_defer_ffn_reduction(forward_batch)
)
self.mlp_reduce_scatter = communicator.should_use_reduce_scatter(forward_batch)
completion = communicator._select_ffn_completion(forward_batch)
self.defer_moe_finalize = completion.defer_moe_finalize
self.fuse_mlp_allreduce = completion.fuse_mlp_allreduce
self.mlp_reduce_scatter = completion.mlp_reduce_scatter
self._leave = completion.leave
self._scope = get_forward().scoped(
fuse_mlp_allreduce=self.fuse_mlp_allreduce,
mlp_reduce_scatter=self.mlp_reduce_scatter,
@@ -1263,26 +1309,8 @@ class FfnExit:
# A deferred MoE finalize handoff, consumed by the next prepare_attn.
assert self.defer_moe_finalize, "unrequested deferred MoE handoff"
return hidden_states, residual
if self.fuse_mlp_allreduce:
return (
UnreducedOutput(
hidden_states,
reduce_and_redistribute=self.communicator._scatter_for_next_layer(
self.forward_batch
),
),
residual,
)
reduce_and_redistribute = self.communicator._reduce_scatter_for_next_layer(
self.forward_batch
)
if reduce_and_redistribute is not None:
return (
UnreducedOutput(
hidden_states, reduce_and_redistribute=reduce_and_redistribute
),
residual,
)
if self._leave is not None:
return self._leave(hidden_states), residual
return self.communicator.postprocess_layer(
hidden_states, residual, self.forward_batch
)
@@ -1797,12 +1825,12 @@ def _redistribute_output_from_moe_cp(
).contiguous()
def _output_to_local_tokens_step(
def _reduce_and_redistribute_output_step(
forward_batch: ForwardBatch, *, allow_reduce_scatter: bool, is_layer_sparse: bool
) -> Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]:
"""How a FULL-layout layer output comes back to this rank's tokens under
attention DP: a reduce-scatter when the FFN left its sum to it, otherwise a
scatter of the reduced output."""
) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]:
"""The reduce-scatter that brings a FULL-layout layer output back to this
rank's tokens under attention DP when the FFN left its sum to it; None when
the FFN reduced the output and only a scatter remains."""
# A MoE block leaves its sum to reduce_scatterv whenever it applies
# (should_skip_post_experts_all_reduce); a dense MLP does only under the
# published mlp_reduce_scatter, which needs allow_reduce_scatter.
@@ -1814,7 +1842,7 @@ def _output_to_local_tokens_step(
and can_use_dp_reduce_scatter()
):
return _reduce_and_redistribute_output_max_len
return _redistribute_output
return None
def _to_local_tokens(
@@ -1828,12 +1856,10 @@ def _to_local_tokens(
def _all_reduce_then_to_local_tokens(
forward_batch: ForwardBatch, hidden_states: torch.Tensor
group: GroupCoordinator, forward_batch: ForwardBatch, hidden_states: torch.Tensor
) -> torch.Tensor:
return _to_local_tokens(
_redistribute_output,
forward_batch,
deferred_post_experts_all_reduce(hidden_states),
_redistribute_output, forward_batch, group.all_reduce(hidden_states)
)
@@ -1917,10 +1943,13 @@ class CommunicateSummableTensorPairFn:
is_layer_sparse: bool = False,
):
local_hidden_states = get_local_dp_buffer(_dp_scatter_group())
step = _output_to_local_tokens_step(
forward_batch,
allow_reduce_scatter=allow_reduce_scatter,
is_layer_sparse=is_layer_sparse,
step = (
_reduce_and_redistribute_output_step(
forward_batch,
allow_reduce_scatter=allow_reduce_scatter,
is_layer_sparse=is_layer_sparse,
)
or _redistribute_output
)
step(local_hidden_states, hidden_states, forward_batch)
return local_hidden_states, residual
@@ -673,16 +673,13 @@ def resolve_fusion_group(*, use_attn_tp_group: bool):
Must match the group the fused residual+LN kernel reduces over; a mismatch
silently reduces across the wrong peers.
"""
from sglang.srt.layers.moe.utils import can_merge_post_experts_all_reduce
from sglang.srt.layers.moe.utils import post_experts_reduction_group
parallel = get_parallel()
if use_attn_tp_group:
return parallel.attn_tp_size, parallel.attn_tp_rank, parallel.attn_tp_group
if can_merge_post_experts_all_reduce():
return parallel.tp_size, parallel.tp_rank, parallel.tp_group
if parallel.moe_ep_size > 1:
return parallel.moe_ep_size, parallel.moe_ep_rank, parallel.moe_ep_group
return parallel.moe_tp_size, parallel.moe_tp_rank, parallel.moe_tp_group
group = post_experts_reduction_group()
return group.world_size, group.rank_in_group, group
def _sync_allreduce_unavailable_across_tp():
+2
View File
@@ -15,6 +15,7 @@ from sglang.srt.layers.moe.utils import (
is_tbo_enabled,
post_experts_all_reduce,
post_experts_output_is_complete,
post_experts_reduction_group,
reduce_moe_output,
should_skip_mlp_all_reduce,
should_skip_post_experts_all_reduce,
@@ -37,6 +38,7 @@ __all__ = [
"deferred_post_experts_all_reduce",
"post_experts_all_reduce",
"post_experts_output_is_complete",
"post_experts_reduction_group",
"reduce_moe_output",
"should_skip_post_experts_all_reduce",
"should_use_dp_reduce_scatterv",
+14 -13
View File
@@ -838,23 +838,24 @@ def post_experts_all_reduce(hidden_states: torch.Tensor) -> torch.Tensor:
return hidden_states
def post_experts_reduction_group():
"""The group one all-reduce of an MoE output runs over: TP when the EP and
MoE-TP reductions merge, otherwise EP, otherwise MoE-TP. The same group
``resolve_fusion_group`` builds the fused workspace on."""
parallel = get_parallel()
if can_merge_post_experts_all_reduce():
return parallel.tp_group
if parallel.moe_ep_size > 1:
return parallel.moe_ep_group
return parallel.moe_tp_group
def deferred_post_experts_all_reduce(hidden_states: torch.Tensor) -> torch.Tensor:
"""Run the post-experts reduction that was deferred to allreduce fusion.
Called when the fused residual+LN kernel cannot service the shape. Reduces
over the same group ``resolve_fusion_group`` builds the workspace on.
Called when the fused residual+LN kernel cannot service the shape.
"""
from sglang.srt.distributed.communication_op import (
moe_expert_parallel_all_reduce,
moe_tensor_model_parallel_all_reduce,
tensor_model_parallel_all_reduce,
)
if can_merge_post_experts_all_reduce():
return tensor_model_parallel_all_reduce(hidden_states)
if get_parallel().moe_ep_size > 1:
return moe_expert_parallel_all_reduce(hidden_states)
return moe_tensor_model_parallel_all_reduce(hidden_states)
return post_experts_reduction_group().all_reduce(hidden_states)
@contextmanager
+6 -2
View File
@@ -427,7 +427,9 @@ class NemotronHMLPLikeDecoderLayer(nn.Module):
):
hidden_states = self.mixer.forward(hidden_states)
if fuse_mlp_allreduce:
hidden_states = UnreducedOutput(hidden_states)
hidden_states = UnreducedOutput(
hidden_states, group=self.layer_communicator.ffn_reduction_group()
)
else:
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
@@ -531,7 +533,9 @@ class NemotronHAttnLikeDecoderLayer(nn.Module):
hidden_states, forward_batch, skip_reduce
)
if fuse_mlp_allreduce:
hidden_states = UnreducedOutput(hidden_states)
hidden_states = UnreducedOutput(
hidden_states, group=get_parallel().tp_group
)
return hidden_states, residual