[Refactor] Select reduction fusion at the consumer (#41551)

This commit is contained in:
Cheng Wan
2026-09-28 20:16:44 -07:00
committed by GitHub
parent cb1fa3c399
commit acffb0d20d
44 changed files with 639 additions and 396 deletions
@@ -40,7 +40,6 @@ from sglang.srt.layers.communicator.boundary import (
from sglang.srt.layers.communicator.layer import (
FfnCompletion,
FfnExit,
FfnExitFusion,
LayerCommunicator,
LayerFacts,
MHCLayerCommunicator,
@@ -91,7 +90,6 @@ __all__ = [
"FUSE_ALLREDUCE_MAX_BATCH_SIZE",
"FfnCompletion",
"FfnExit",
"FfnExitFusion",
"FusedMlpInput",
"HandoffOutput",
"LayerCommunicator",
@@ -49,6 +49,7 @@ from sglang.srt.layers.communicator.ops import (
_read_input,
move_rows,
tp_reduce_scatter,
tp_slice,
)
from sglang.srt.layers.communicator.residual import (
LayerResidual,
@@ -551,6 +552,7 @@ class StageEntry(msgspec.Struct, frozen=True):
capture_move: Optional[Callable] = None
capture_move_allocates: bool = False
capture_preserves_residual: Optional[Callable] = None
input_sum: Optional[SumGroup] = None
class BoundarySteps(msgspec.Struct, frozen=True):
@@ -724,6 +726,20 @@ def _bind_consumer(
cp_moves=cp_moves,
enters_stack=enters_stack,
)
completed_step = None
if edge.produced.always_leaves:
completed_step, _, _ = _select_input_steps(
msgspec.structs.replace(edge.produced, always_leaves=False),
residual=edge.residual,
residual_to=edge.residual_to,
need=edge.need,
adds_plainly=adds_plainly,
fusions=(),
force_layernorm_before_gather=force_layernorm_before_gather,
residual_joins_sum=False,
cp_moves=cp_moves,
enters_stack=enters_stack,
)
# A written stream entering the first physical layer must not run enter
# again (e.g. MHC expansion). Both alternatives are bound at construction.
written_step = None
@@ -747,7 +763,8 @@ def _bind_consumer(
step=step,
adds_plainly=adds_plainly,
carried_fusions=carried_fusions if plain else (),
owes_by_construction=edge.produced.always_leaves,
expected_sum=edge.produced.group if edge.produced.always_leaves else None,
completed_step=completed_step,
written_step=written_step,
),
input_move=input_move,
@@ -826,12 +843,18 @@ def _select_boundary_steps(
handoff=attention_handoff,
capture_move=into_attention.capture_move,
capture_move_allocates=into_attention.capture_move_allocates,
input_sum=edges.into_attention.produced.group
if edges.into_attention.produced.always_leaves
else None,
),
ffn=StageEntry(
prepare=into_ffn.prepare,
input_rows=into_ffn.input_rows,
input_move=into_ffn.input_move,
fused=into_ffn.fused,
input_sum=edges.into_ffn.produced.group
if edges.into_ffn.produced.always_leaves
else None,
),
ffn_output=edges.out_of_ffn.produced,
ffn_output_move=out_of_ffn.output_move,
@@ -895,17 +918,19 @@ def _select_input_steps(
if residual == residual_to:
# The residual stays on each rank's slice while the stage takes the
# rows around it (MHC on an input-scattered batch).
if gathered or owes is not SumGroup.ATTN_TP:
if gathered or owes not in (None, SumGroup.ATTN_TP):
raise NotImplementedError(f"{produced=} {residual=} {need=}")
return (
partial(_mlp_input_on_residual_shard, read=read),
partial(
_mlp_input_on_residual_shard, read=read, reduces=owes is not None
),
(),
None,
)
# A reduce-scatter completes the TP sum onto each rank's slice, which
# the stage takes: its group is the TP group without attention DP or CP.
if (
owes is not SumGroup.TP
owes not in (None, SumGroup.TP)
or residual.sharded
or TokenAxis.ATTN_TP_SCATTER not in need.gathers_itself
or residual_to.sharded != {TokenAxis.ATTN_TP_SCATTER}
@@ -914,7 +939,7 @@ def _select_input_steps(
return (
partial(
_read_input,
layer_input=tp_reduce_scatter,
layer_input=tp_reduce_scatter if owes is not None else tp_slice,
enters_stack=enters_stack,
read=read,
),
@@ -974,7 +999,17 @@ def _select_input_steps(
if not gathered:
if owes is None:
if gathers_residual:
raise NotImplementedError(f"{produced=} {residual=} {need=}")
return (
partial(
_mlp_input_without_dp,
gathers_residual=True,
fusions=(),
group=None,
read=read,
),
(),
None,
)
return (
partial(
_read_input,
@@ -14,7 +14,6 @@ from typing import Callable, Optional, Sequence
import torch
from sglang.srt.layers.communicator import (
FfnExitFusion,
FusedMlpInput,
HandoffOutput,
LayerCommunicator,
@@ -201,33 +200,27 @@ class CuteDSLFusion:
construction. At the attention input: the MoE finalize + AR + add + norm of
a handoff the previous layer left, and the AR + add + norm of a sum it left.
At the FFN input: the attention output's AR + add + norm. At the FFN exit:
handing the MoE finalize, or the all-reduce, to the next layer's kernel.
install_cutedsl_fusion gives it the model's service and chooses the exit
from the layer after this one; until then no kernel takes a batch."""
handing off a deferred MoE finalize with its unfused completion.
install_cutedsl_fusion supplies the service and producer capability;
consumer eligibility is checked only where the output is consumed."""
def __init__(self) -> None:
self.service: CuteDSLFusionService | None = None
# The FFN exit's choices, from what install() knows of the next layer.
# Producer capabilities; consumer kernel selection is independent.
self.hands_off_finalize = False
self.next_input_absorbs = False
self.requires_local_reduction = False
def install(
self,
service: CuteDSLFusionService,
*,
hands_off_finalize: bool,
next_input_absorbs: bool,
output_is_replicated: bool,
) -> None:
"""Take the model's fusion service and choose the FFN exit's kernels:
the MoE may hand off its finalize when it can and something after it
takes the handoff (``hands_off_finalize``); the FFN may leave its
all-reduce to the next layer's AR + norm when that layer has one, unless
a replicated output follows the reduction (moving it would scale that
by tp)."""
"""Install the shared workspace and this producer's capabilities."""
self.service = service
self.hands_off_finalize = hands_off_finalize
self.next_input_absorbs = next_input_absorbs and not output_is_replicated
self.requires_local_reduction = output_is_replicated
def attention_input(self, layer: LayerCommunicator) -> tuple:
return (
@@ -254,12 +247,6 @@ class CuteDSLFusion:
)
return ()
def ffn_exit(self, layer: LayerCommunicator) -> tuple:
return (
partial(self._defer_moe_finalize, layer),
partial(self._absorb_all_reduce, layer),
)
def _finalize_output_and_update_and_read_residual(
self, layer, owed, residual, forward_batch, post_residual_addition
):
@@ -330,26 +317,28 @@ class CuteDSLFusion:
and not get_exec().comm.enable_quant_communications
)
def _defer_moe_finalize(
self, layer, forward_batch: ForwardBatch
) -> Optional[FfnExitFusion]:
"""The MoE hands its unfinalized output to the next layer's finalize +
all-reduce + norm."""
if self.hands_off_finalize and self._should_use_finalize(
layer, forward_batch, int(forward_batch.input_ids.shape[0])
):
return FfnExitFusion.DEFER_MOE_FINALIZE
return None
def can_defer_all_reduce(self, layer, forward_batch: ForwardBatch) -> bool:
"""Preserve the producer's fused sum path when this workspace is usable.
def _absorb_all_reduce(
self, layer, forward_batch: ForwardBatch
) -> Optional[FfnExitFusion]:
"""The next layer's AR + norm takes the post-experts all-reduce."""
if self.next_input_absorbs and self._can_consume_post_moe_all_reduce(
layer, forward_batch, int(forward_batch.input_ids.shape[0])
):
return FfnExitFusion.NEXT_INPUT
return None
The next consumer still selects its kernel and completes the sum with
an ordinary all-reduce if that kernel declines the actual input.
"""
return (
self._common_eligible(
layer, forward_batch, int(forward_batch.input_ids.shape[0])
)
and not get_exec().comm.enable_quant_communications
)
def can_defer_finalize(self, layer, forward_batch: ForwardBatch) -> bool:
"""Whether this producer can emit a handoff with an unfused fallback."""
return (
self.hands_off_finalize
and not self.requires_local_reduction
and self._should_use_finalize(
layer, forward_batch, int(forward_batch.input_ids.shape[0])
)
)
def _common_eligible(self, layer, forward_batch: ForwardBatch, m: int) -> bool:
parallel = get_parallel()
@@ -383,7 +372,6 @@ def install_cutedsl_fusion(
rms_epsilon: float,
can_defer_finalize: _LayerPredicate,
requires_local_reduction: _LayerPredicate | None = None,
final_norm_consumes_handoff: bool = False,
label: str,
) -> CuteDSLFusionService | None:
"""One shared workspace handle per fusion-enabled layer, or None.
@@ -419,23 +407,15 @@ def install_cutedsl_fusion(
rms_epsilon=rms_epsilon,
)
hands_off = 0
for index, layer in enumerate(layers):
for layer in layers:
fusion = _fusion_of(layer)
if fusion is None:
continue
successor = layers[index + 1] if index + 1 < len(layers) else None
if successor is None:
has_consumer = final_norm_consumes_handoff
else:
has_consumer = _fusion_of(successor) is not None
hands_off_finalize = bool(can_defer_finalize(layer)) and has_consumer
hands_off_finalize = bool(can_defer_finalize(layer))
hands_off += hands_off_finalize
fusion.install(
service,
hands_off_finalize=hands_off_finalize,
# False on the last layer: the final norm does not all-reduce.
next_input_absorbs=successor is not None
and _fusion_of(successor) is not None,
output_is_replicated=requires_local_reduction is not None
and bool(requires_local_reduction(layer)),
)
+124 -93
View File
@@ -14,7 +14,6 @@
"""The communicator a model layer uses."""
from dataclasses import dataclass
from enum import Enum, auto
from functools import cached_property, partial
from typing import Callable, Optional, Protocol, Tuple, Union
@@ -49,7 +48,6 @@ from sglang.srt.layers.communicator.layout import (
SumGroup,
TokenAxis,
_batch_shards_over_cp,
_batch_size,
_ffn_has_tokens,
_gathers_over_attention_cp,
_generic_prefill_cp_shards_tokens,
@@ -86,13 +84,13 @@ from sglang.srt.layers.communicator.residual.access import (
written,
)
from sglang.srt.layers.communicator.residual.add_norm import (
ADD,
PLAIN_RESIDUAL,
aiter_all_reduce_fusion_enabled_for,
apply_aiter_all_reduce_fusion,
apply_flashinfer_allreduce_fusion,
)
from sglang.srt.layers.communicator.residual.mhc import MHCState
from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.srt.layers.communicator.residual.stream import DeclaredSum, ResidualStream
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
is_enable_moe_cp_allgather,
@@ -156,11 +154,39 @@ class LayerFacts:
)
def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool:
"""Whether the next layer's all-reduce is the one the FFN would have run
itself, as the MoE declares it (post_experts_sum_is_one_all_reduce), on a
batch the FFN runs."""
return _ffn_has_tokens(forward_batch) and post_experts_sum_is_one_all_reduce()
def _can_defer_ffn_reduction(forward_batch: ForwardBatch, boundary=None) -> bool:
"""Admit ordinary single-sum outputs and the existing fused LoRA path.
A fused-kernel fallback completes the partial-output contract selected
before the producer ran, without recomputing the producer's LoRA path.
"""
if not _ffn_has_tokens(forward_batch):
return False
if post_experts_sum_is_one_all_reduce():
return True
# LoRA-B is replicated and linear. TP1 shared experts add on rank zero
# when the sum is deferred. Preserve these fused paths, but keep their
# producer-side reduction order when no fused consumer is enabled.
if not (
(get_lora().enable_lora or envs.SGLANG_SHARED_EXPERT_TP1.get())
and get_moe_a2a_backend().is_none()
and not get_exec().comm.enable_quant_communications
and post_experts_reduction_group() is get_parallel().tp_group
):
return False
if (
boundary is not None
and boundary.fusions is not None
and boundary.fusions.can_defer_all_reduce(boundary, forward_batch)
):
return True
if apply_flashinfer_allreduce_fusion(forward_batch.input_ids.shape[0]):
return True
# Aiter also checks width and bytes, so use the actual residual storage.
residual = forward_batch.residual_stream.residual
return residual is not None and apply_aiter_all_reduce_fusion(
residual, forward_batch
)
class StageCommunicator:
@@ -205,12 +231,20 @@ class StageCommunicator:
stream = residual_batch.current(forward_batch)
if residual is not stream:
raise RuntimeError("residual alias belongs to a different invocation")
if stream is not None:
if stream.pending is not None:
call["update"] = stream.pending.update
if stream.pending is None and stream.residual is not None:
call["written"] = True
hidden_states, residual = stream.input(hidden_states)
if (
stream.pending is None
and stream.residual is None
and entry.input_sum is not None
):
hidden_states = stream.leave(
hidden_states, ADD, declared_sum=entry.input_sum
)
if stream.pending is not None:
call["update"] = stream.pending.update
call["pending"] = stream.pending
if stream.pending is None and stream.residual is not None:
call["written"] = True
hidden_states, residual = stream.input(hidden_states)
hidden_states, residual = entry.prepare(
hidden_states, residual, forward_batch, self.norm, context, **call
)
@@ -239,9 +273,15 @@ class LayerFusions(Protocol):
def ffn_input(self, layer: "LayerCommunicator") -> Tuple["FusedMlpInput", ...]: ...
def ffn_exit(
self, layer: "LayerCommunicator"
) -> Tuple[Callable[[ForwardBatch], Optional["FfnExitFusion"]], ...]: ...
requires_local_reduction: bool
def can_defer_all_reduce(
self, layer: "LayerCommunicator", forward_batch: ForwardBatch
) -> bool: ...
def can_defer_finalize(
self, layer: "LayerCommunicator", forward_batch: ForwardBatch
) -> bool: ...
class LayerCommunicator:
@@ -255,7 +295,6 @@ class LayerCommunicator:
# Communicators built without __init__ (e.g. test doubles) publish no LoRA
# layout and try no fused kernel at the FFN exit.
_publish_lora_layout: bool = False
_ffn_exit_fusions: Tuple[Callable, ...] = ()
# A plain residual unless the layer is built with its own.
_residual: LayerResidual = PLAIN_RESIDUAL
# The fused kernels a backend gives the layer, if any.
@@ -303,8 +342,6 @@ class LayerCommunicator:
)
# The fused kernels every batch's attention input tries first.
self._attn_input_fusions = self._select_attn_input_fusions()
# The fused kernels of the next layer's input the FFN exit tries first.
self._ffn_exit_fusions = self._select_ffn_exit_fusions()
self._speculative_algo = SpeculativeAlgorithm.from_string(
get_spec().speculative_algorithm
)
@@ -433,6 +470,9 @@ class LayerCommunicator:
handoff=None if is_ffn else _hand_qkv_hook_its_input,
fused=into.fused,
capture_move=into.capture_move,
input_sum=into_edge.produced.group
if into_edge.produced.always_leaves
else None,
)
self._steps = BoundarySteps(
attention=None if is_ffn else entry,
@@ -641,7 +681,11 @@ class LayerCommunicator:
def snapshot(self, hidden_states, residual, *, at_input: bool = False):
group = None
if residual is not None and self.stage_edges is not None:
if (
residual is not None
and not isinstance(residual, ResidualStream)
and self.stage_edges is not None
):
edge = self.stage_edges[0 if at_input else 1]
if edge.produced.always_leaves:
group = _sum_group(edge.produced.group)
@@ -663,7 +707,12 @@ class LayerCommunicator:
if storage is not None and storage.shape[0] == 0:
return hidden_states, None
hidden_states = (
residual.complete(hidden_states)
(
hidden_states
if residual.pending is not None
and isinstance(residual.pending.owed, DeclaredSum)
else residual.complete(hidden_states)
)
if isinstance(residual, ResidualStream)
else reduce_output(hidden_states)
)
@@ -675,8 +724,10 @@ class LayerCommunicator:
residual_in_hidden=self._residual.ffn_update.at_producer,
allow_missing_residual=allow_missing_residual,
)
steps = self._batch_steps(forward_batch)
entry = steps.attention or steps.ffn
hidden_states, stream = ResidualStream.arrive(
hidden_states, residual, self._residual.ffn_update
hidden_states, residual, ADD, declared_sum=entry.input_sum
)
forward_batch.residual_stream = stream
return hidden_states, stream
@@ -934,7 +985,9 @@ class LayerCommunicator:
if isinstance(residual, ResidualStream) and self.stage_edges is None:
# The paired boundary records the attention contribution once.
hidden_states = residual.leave(
hidden_states, self._residual.attention_update
hidden_states,
self._residual.attention_update,
declared_sum=steps.ffn.input_sum,
)
return self.ffn.prepare(
hidden_states,
@@ -969,16 +1022,34 @@ class LayerCommunicator:
residual,
stream,
self._batch_steps(forward_batch).ffn_output.update,
self._declared_ffn_sum(
self._batch_steps(forward_batch),
self._ffn_leaves_sum_to_reduce_scatter(
forward_batch, self._postprocess_dp_step(forward_batch)
),
),
)
@staticmethod
def _leave_ffn_output(hidden_states, residual, stream, update):
def _leave_ffn_output(hidden_states, residual, stream, update, declared_sum=None):
if stream is None:
return hidden_states, residual
stream.residual = residual
if update.at_producer:
return stream.write(hidden_states), stream
return stream.leave(hidden_states, update), stream
return stream.leave(hidden_states, update, declared_sum=declared_sum), stream
@staticmethod
def _declared_ffn_sum(steps, skipped_reduction):
# The input-scattered path leaves the sum for the next input's TP
# reduce-scatter. Other skip paths complete it in the output move.
if (
skipped_reduction
and not steps.returns_over_dp
and not steps.ffn_output_move_completes_sum
):
return SumGroup.TP
return None
def _local_token_move_can_go_to_next_layer(
self, forward_batch: ForwardBatch
@@ -1042,14 +1113,13 @@ class LayerCommunicator:
mlp_reduce_scatter=mlp_reduce_scatter,
complete=complete_now,
)
fusion = next(
filter(None, (fused(forward_batch) for fused in self._ffn_exit_fusions)),
None,
defer_moe_finalize = (
self.fusions is not None
and self.fusions.can_defer_finalize(self, forward_batch)
)
defer_moe_finalize = fusion is FfnExitFusion.DEFER_MOE_FINALIZE
# A fused kernel that takes the sum, a handoff included, skips the
# post-experts all-reduce.
fuse_mlp_allreduce = fusion is not None or self._ffn_sum_moves_to_next_layer(
# Producers declare remaining work independently of the kernel chosen
# by the consumer. Every handoff also carries an unfused completion.
fuse_mlp_allreduce = defer_moe_finalize or self._ffn_sum_moves_to_next_layer(
forward_batch, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step
)
if fuse_mlp_allreduce:
@@ -1205,28 +1275,12 @@ class LayerCommunicator:
return stream.leave(output, update), stream
return output, residual
def _select_ffn_exit_fusions(
self,
) -> Tuple[Callable[[ForwardBatch], Optional["FfnExitFusion"]], ...]:
"""The fused kernels of the next layer's input that may take this layer's
FFN sum, in the order they are tried. Each returns what the exit does
when its kernel takes this batch, or None. A backend's come first."""
given = self.fusions.ffn_exit(self) if self.fusions else ()
return (*given, self._next_input_norm_takes_ffn_sum)
def _next_input_norm_takes_ffn_sum(
self, forward_batch: ForwardBatch
) -> Optional["FfnExitFusion"]:
"""The aiter / flashinfer AR + add + norm of the next layer's input."""
if self.should_fuse_mlp_allreduce_with_next_layer(forward_batch):
return FfnExitFusion.NEXT_INPUT
return None
def _ffn_sum_can_move_to_next_layer(self, forward_batch: ForwardBatch) -> bool:
# Under the MoE-CP all-gather the fusion path would skip postprocess_layer
# and its MoE-CP scatter, leaving hidden_states longer than the residual.
if (
is_enable_moe_cp_allgather()
(self.fusions is not None and self.fusions.requires_local_reduction)
or is_enable_moe_cp_allgather()
or not self._batch_steps(forward_batch).ffn_sum_is_movable
):
return False
@@ -1252,29 +1306,6 @@ class LayerCommunicator:
return not get_attn_tp_context().input_scattered
# NOTE: This function will cause torch recompilation
def should_fuse_mlp_allreduce_with_next_layer(
self, forward_batch: ForwardBatch
) -> bool:
if not self._ffn_sum_can_move_to_next_layer(forward_batch):
return False
batch_size = _batch_size(forward_batch)
return (
(
apply_flashinfer_allreduce_fusion(batch_size)
or (
_use_aiter
and batch_size > 0
and get_parallel().tp_size != 6
and not is_dp_attention_enabled()
and get_moe_a2a_backend().is_none()
and aiter_all_reduce_fusion_enabled_for(forward_batch.forward_mode)
)
)
and (not self.is_last_layer)
and (self._context.tp_size > 1)
)
def _ffn_sum_moves_to_next_layer(
self,
forward_batch: ForwardBatch,
@@ -1289,7 +1320,7 @@ class LayerCommunicator:
self._context.tp_size > 1
and not self.is_last_layer
and self._ffn_sum_can_move_to_next_layer(forward_batch)
and _unfused_completion_matches_the_ffn(forward_batch)
and _can_defer_ffn_reduction(forward_batch, self)
and not mlp_reduce_scatter
# Under attention DP the next layer must also run postprocess's
# scatter back to this rank's tokens, and nothing more.
@@ -1303,16 +1334,6 @@ class LayerCommunicator:
)
class FfnExitFusion(Enum):
"""What an FFN exit does when a fused kernel of the next layer's input takes
its sum."""
# The MoE hands its unfinalized output on; the next input finalizes it.
DEFER_MOE_FINALIZE = auto()
# The FFN leaves its all-reduce to the next input norm.
NEXT_INPUT = auto()
class FfnCompletion(msgspec.Struct, frozen=True):
"""One FFN's reduction decision. The flags are published while the FFN runs;
``complete(hidden_states, residual)`` then either wraps the output for the
@@ -1337,17 +1358,20 @@ class MixerExit:
``with`` block ``fuse_mlp_allreduce`` on ``get_forward()`` tells its
row-parallel output projection to skip the all-reduce. It skips when the
stage's output always leaves its sum (to an FFN stage, which completes it in
its input), and when it may leave it and the fused kernel takes it into the
next input norm."""
its input), and when its output declaration permits deferring the sum. The
consumer decides whether to fuse its completion with the input norm."""
__slots__ = ("skips_reduction", "_hands_on", "_scope", "_update")
__slots__ = ("skips_reduction", "_hands_on", "_scope", "_update", "_declared_sum")
def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch):
produced = communicator._batch_steps(forward_batch).ffn_output
self._update = produced.update
self._declared_sum = produced.group if produced.always_leaves else None
self._hands_on = (
produced.leaves_for_next_layer
and communicator.should_fuse_mlp_allreduce_with_next_layer(forward_batch)
and communicator._context.tp_size > 1
and not is_dp_attention_enabled()
and communicator._ffn_sum_can_move_to_next_layer(forward_batch)
)
self.skips_reduction = produced.always_leaves or self._hands_on
self._scope = get_forward().scoped(fuse_mlp_allreduce=self.skips_reduction)
@@ -1367,7 +1391,9 @@ class MixerExit:
hidden_states, group=get_parallel().tp_group
)
if isinstance(residual, ResidualStream):
return residual.leave(hidden_states, self._update), residual
return residual.leave(
hidden_states, self._update, declared_sum=self._declared_sum
), residual
return hidden_states
@@ -1384,6 +1410,7 @@ class FfnExit:
"mlp_reduce_scatter",
"_complete",
"_update",
"_declared_sum",
"_scope",
)
@@ -1395,7 +1422,11 @@ class FfnExit:
self.fuse_mlp_allreduce = completion.fuse_mlp_allreduce
self.mlp_reduce_scatter = completion.mlp_reduce_scatter
self._complete = completion.complete
self._update = communicator._batch_steps(forward_batch).ffn_output.update
steps = communicator._batch_steps(forward_batch)
self._update = steps.ffn_output.update
self._declared_sum = communicator._declared_ffn_sum(
steps, completion.mlp_reduce_scatter
)
self._scope = get_forward().scoped(
fuse_mlp_allreduce=self.fuse_mlp_allreduce,
mlp_reduce_scatter=self.mlp_reduce_scatter,
@@ -1422,7 +1453,7 @@ class FfnExit:
else:
hidden_states, residual = self._complete(hidden_states, residual)
return self.communicator._leave_ffn_output(
hidden_states, residual, stream, self._update
hidden_states, residual, stream, self._update, self._declared_sum
)
+29 -6
View File
@@ -52,6 +52,7 @@ from sglang.srt.layers.communicator.output import (
)
from sglang.srt.layers.communicator.residual import StageRead, StageUpdate
from sglang.srt.layers.communicator.residual.add_norm import ADD, NORM_READ
from sglang.srt.layers.communicator.residual.stream import DeclaredSum
from sglang.srt.layers.dp_attention import (
attn_tp_all_gather_into_tensor,
attn_tp_reduce_scatter_tensor,
@@ -102,6 +103,14 @@ def tp_reduce_scatter(
return output, residual
def tp_slice(hidden_states, residual, context):
"""The rows reduce-scatter would return, when the sum is already complete."""
hidden_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank].clone()
if residual is not None:
residual = residual.tensor_split(context.tp_size)[context.tp_rank]
return hidden_states, residual
class CommunicateSimpleFn:
@staticmethod
def _trivial(
@@ -263,7 +272,7 @@ def _mlp_input_without_dp(
hidden_states = _mlp_input_reduce_output(
hidden_states, forward_batch, may_quantize=update.adds_plainly
)
else:
elif group is SumGroup.TP:
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
if _is_npu and context.cache is not None:
_ = prepare_weight_cache(hidden_states, context.cache)
@@ -411,7 +420,9 @@ def _consumer_step(
step: Callable,
adds_plainly: bool,
carried_fusions: Tuple[Callable, ...],
owes_by_construction: bool = False,
expected_sum: Optional[SumGroup] = None,
completed_step: Optional[Callable] = None,
pending=None,
written_step: Optional[Callable] = None,
written: bool = False,
update: StageUpdate = ADD,
@@ -422,18 +433,26 @@ def _consumer_step(
complete value. Under attention DP the reduction back to this rank's tokens
comes first; otherwise one of ``carried_fusions`` may complete it with the
residual add and the read, and an empty batch has nothing to sum. An input
that owes its sum by construction (``owes_by_construction``) is that sum's
only carrier. ``call`` is what the stage's read takes (the attention's
whose declared sum is already completed uses ``completed_step`` instead;
a pending declared sum must match ``expected_sum``. ``call`` is what the stage's read takes (the attention's
``quant_format`` and ``post_residual_addition``)."""
if residual is not None and update.adds_plainly != adds_plainly:
raise RuntimeError("producer update does not match the boundary's capability")
if pending is not None:
if isinstance(pending.owed, DeclaredSum):
if pending.owed.group is not expected_sum:
raise RuntimeError("producer sum does not match the input declaration")
elif expected_sum is not None:
if pending.owed is not None:
raise RuntimeError("a declared input sum carried another completion")
step = completed_step
if written and written_step is not None:
step = written_step
if not isinstance(hidden_states, torch.Tensor):
owed = hidden_states
if residual is None:
raise RuntimeError(f"{type(owed).__name__} requires residual input")
if owes_by_construction:
if expected_sum is not None:
raise RuntimeError(
f"an input that owes its sum by construction arrived as "
f"{type(owed).__name__}"
@@ -538,6 +557,7 @@ def _mlp_input_on_residual_shard(
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
reduces: bool = True,
read: StageRead,
update: StageUpdate,
):
@@ -548,7 +568,10 @@ def _mlp_input_on_residual_shard(
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
shard = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
get_parallel().tp_group.reduce_scatter_tensor(shard, hidden_states)
if reduces:
get_parallel().tp_group.reduce_scatter_tensor(shard, hidden_states)
else:
shard = shard.clone()
shard, residual = read.update_and_read(update, shard, residual, layernorm)
attn_tp_all_gather_into_tensor(hidden_states, shard)
return hidden_states, residual
@@ -82,6 +82,7 @@ def finish_layer_stack(
forward_batch: ForwardBatch,
*,
final_norm_takes_handoff: bool = False,
preserve_declared: bool = False,
) -> Tuple[Union[torch.Tensor, HandoffOutput], Optional[torch.Tensor]]:
"""Complete what this layer left for a next layer. Call it on the last
layer of this rank before its output reaches the final norm, the next
@@ -92,7 +93,11 @@ def finish_layer_stack(
stream = forward_batch.residual_stream
if residual is not stream:
raise RuntimeError("residual alias belongs to a different invocation")
output = stream.finish(hidden_states, takes_handoff=final_norm_takes_handoff)
output = stream.finish(
hidden_states,
takes_handoff=final_norm_takes_handoff,
preserve_declared=preserve_declared,
)
forward_batch.residual_stream = None
return output
if final_norm_takes_handoff and isinstance(hidden_states, HandoffOutput):
@@ -19,6 +19,7 @@ import msgspec
import torch
from sglang.srt.distributed import GroupCoordinator
from sglang.srt.layers.communicator.layout import SumGroup, _sum_group
from sglang.srt.layers.communicator.output import HandoffOutput, UnreducedOutput
from sglang.srt.layers.communicator.residual import StageUpdate
@@ -35,12 +36,21 @@ class CarriedSum(msgspec.Struct, frozen=True):
return self.group.all_reduce(value)
class DeclaredSum(msgspec.Struct, frozen=True):
"""A sum every output of this producer owes to its declared input edge."""
group: SumGroup
def complete(self, value):
return _sum_group(self.group).all_reduce(value)
class Contribution(msgspec.Struct):
"""The sole owner of an output's value, update and remaining work."""
value: Optional[torch.Tensor]
update: StageUpdate
owed: Union[CarriedSum, HandoffOutput, None] = None
owed: Union[CarriedSum, DeclaredSum, HandoffOutput, None] = None
def for_boundary(self):
# These forms are private inputs to the existing fused-kernel adapters.
@@ -53,7 +63,7 @@ class Contribution(msgspec.Struct):
return self.value
def complete(self):
if isinstance(self.owed, CarriedSum):
if isinstance(self.owed, (CarriedSum, DeclaredSum)):
value = self.owed.complete(self.value)
elif isinstance(self.owed, HandoffOutput):
value = self.owed.complete()
@@ -84,22 +94,26 @@ class ResidualStream:
self.pending = None
@classmethod
def arrive(cls, hidden, residual, update):
def arrive(cls, hidden, residual, update, *, declared_sum=None):
"""Rebuild an explicit tensor handoff, including each TBO microbatch."""
stream = cls(residual)
if residual is None:
return stream.write(hidden), stream
return stream.leave(hidden, update), stream
return stream.leave(hidden, update, declared_sum=declared_sum), stream
def write(self, residual):
self.residual = residual
self.pending = None
return residual
def leave(self, output, update):
def leave(self, output, update, *, declared_sum=None):
if self.pending is not None:
raise RuntimeError("cannot replace an unconsumed producer contribution")
if isinstance(output, UnreducedOutput):
if declared_sum is not None:
if not isinstance(output, torch.Tensor):
raise RuntimeError("a declared sum cannot carry another completion")
contribution = Contribution(output, update, DeclaredSum(declared_sum))
elif isinstance(output, UnreducedOutput):
contribution = Contribution(
output.partial,
update,
@@ -151,7 +165,7 @@ class ResidualStream:
return hidden.clone()
if pending.owed is None:
value = pending.value
elif isinstance(pending.owed, CarriedSum):
elif isinstance(pending.owed, (CarriedSum, DeclaredSum)):
value = pending.owed.complete(pending.value.clone())
else:
raise NotImplementedError("a finalize handoff requires main-output capture")
@@ -159,10 +173,12 @@ class ResidualStream:
value = group.all_reduce(value.clone())
return value.clone() if self.residual is None else value + self.residual
def finish(self, hidden, *, takes_handoff=False):
def finish(self, hidden, *, takes_handoff=False, preserve_declared=False):
self.check(hidden)
if self.pending is None:
return hidden, None
if preserve_declared and isinstance(self.pending.owed, DeclaredSum):
return self.pending.value, self.residual
if takes_handoff and isinstance(self.pending.owed, HandoffOutput):
return self.pending.owed, self.residual
return self.pending.complete(), self.residual
+4 -1
View File
@@ -848,7 +848,10 @@ class BailingMoEModel(nn.Module):
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
@@ -1008,7 +1008,10 @@ class BailingMoELinearModel(nn.Module):
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
+4 -1
View File
@@ -1401,7 +1401,10 @@ class BailingMoELinearModel(nn.Module):
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
+5 -3
View File
@@ -2979,7 +2979,7 @@ class DeepseekV2Model(nn.Module):
install_cutedsl_fusion(
# PP pads self.layers with PPMissingLayer, which has no communicator.
# A stage's last layer has no successor, so it keeps its reduction.
# A pipeline exit completes any handoff before sending its tensors.
self.layers[self.start_layer : self.end_layer],
hidden_size=config.hidden_size,
top_k=config.num_experts_per_tok,
@@ -2999,7 +2999,6 @@ class DeepseekV2Model(nn.Module):
if isinstance(layer.mlp, DeepseekV2MoE)
else layer.mlp.tp_size == 1
),
final_norm_consumes_handoff=False,
label="DeepSeek-V3/GLM",
)
@@ -3135,7 +3134,10 @@ class DeepseekV2Model(nn.Module):
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
proxy_tensors = {
+6 -1
View File
@@ -584,7 +584,12 @@ class GigaChat35Model(nn.Module):
hidden_states, residual = self.layers[
self.end_layer - 1
].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch)
].layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
+4 -1
View File
@@ -1134,7 +1134,10 @@ class Glm4MoeModel(nn.Module):
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
+4 -1
View File
@@ -833,7 +833,10 @@ class Glm4MoeLiteModel(nn.Module):
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
+4 -1
View File
@@ -1191,7 +1191,10 @@ class Glm5NextModel(nn.Module):
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
if self.config.mhc:
+4 -1
View File
@@ -732,7 +732,10 @@ class GptOssModel(nn.Module):
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
+4 -1
View File
@@ -611,7 +611,10 @@ class LagunaModel(nn.Module):
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
+6 -1
View File
@@ -766,7 +766,12 @@ class LLaDA2MoeModel(nn.Module):
hidden_states, residual = self.layers[
self.end_layer - 1
].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch)
].layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
+4 -1
View File
@@ -1081,7 +1081,10 @@ class MiMoV2Model(nn.Module):
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
# A draft targeting the final layer ("after layer
+4 -1
View File
@@ -1184,7 +1184,10 @@ class MiniMaxM2Model(nn.Module):
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
+4 -1
View File
@@ -1522,7 +1522,10 @@ class MiniMaxM3Model(nn.Module):
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
+4 -1
View File
@@ -760,7 +760,10 @@ class NemotronHModel(nn.Module):
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
+4 -1
View File
@@ -1220,7 +1220,10 @@ class Qwen2MoeModel(nn.Module):
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
+1 -2
View File
@@ -1746,8 +1746,6 @@ class Qwen3_5ForCausalLM(nn.Module):
rms_epsilon=config.rms_norm_eps,
# Every layer was checked above.
can_defer_finalize=lambda layer: True,
# The final GemmaRMSNorm closes out the last layer's handoff.
final_norm_consumes_handoff=True,
label="Qwen3.5",
)
@@ -1870,6 +1868,7 @@ class Qwen3_5ForCausalLM(nn.Module):
# The final norm below finalizes a deferred MoE output in its kernel.
final_norm_takes_handoff=self.flashinfer_mnnvl_cutedsl_fusion is not None
and self.pp_group.is_last_rank,
preserve_declared=not self.pp_group.is_last_rank,
)
# Return intermediate tensors for pipeline parallelism
+6 -1
View File
@@ -1258,7 +1258,12 @@ class Qwen3LLMModel(Qwen3Model):
hidden_states, residual = self.layers[
self.end_layer - 1
].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch)
].layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
# Handle deepstack for the last processed layer (HF-order path only).
last_deepstack = (
+4 -1
View File
@@ -158,7 +158,10 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
+4 -1
View File
@@ -1167,7 +1167,10 @@ class SarvamMLAModel(nn.Module):
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
+6 -1
View File
@@ -421,7 +421,12 @@ class SDARModel(nn.Module):
hidden_states, residual = self.layers[
self.end_layer - 1
].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch)
].layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
+4 -1
View File
@@ -486,7 +486,10 @@ class SDARMoeModel(nn.Module):
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
+4 -1
View File
@@ -716,7 +716,10 @@ class Step3p5Model(nn.Module):
# break
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
+6 -1
View File
@@ -1662,7 +1662,12 @@ class XllmModel(nn.Module):
hidden_states, residual = self.layers[
self.end_layer - 1
].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch)
].layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{