[Refactor] Move CuTe DSL-fused layers onto the declared boundaries and FFN-exit kernel entries (#41431)

This commit is contained in:
Cheng Wan
2026-09-27 16:51:48 -07:00
committed by GitHub
parent 8c43c667cb
commit f0ecf15c84
8 changed files with 375 additions and 125 deletions
+71 -24
View File
@@ -761,14 +761,28 @@ class UnreducedOutput(msgspec.Struct, frozen=True):
reduce_and_redistribute: Optional[Callable[[torch.Tensor], torch.Tensor]] = None
class HandoffOutput(msgspec.Struct, frozen=True):
"""A layer output that still owes work only its producer knows how to do (a
MoE's finalize and sum), left for the next layer's input. A fused kernel
there may do that work together with its own; anything else passes it
through reduce_output(), which calls ``complete()``."""
def complete(self) -> torch.Tensor:
"""Do the owed work, unfused, and return the complete output."""
raise NotImplementedError
def reduce_output(
hidden_states: Union[torch.Tensor, UnreducedOutput, None],
hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput, None],
) -> Optional[torch.Tensor]:
"""Run the reduction an UnreducedOutput still owes; pass anything else through."""
"""Run the work an UnreducedOutput or a HandoffOutput still owes; pass
anything else through."""
if isinstance(hidden_states, UnreducedOutput):
if hidden_states.reduce_and_redistribute is not None:
return hidden_states.reduce_and_redistribute(hidden_states.partial)
return hidden_states.group.all_reduce(hidden_states.partial)
if isinstance(hidden_states, HandoffOutput):
return hidden_states.complete()
return hidden_states
@@ -802,11 +816,10 @@ def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool:
class LayerCommunicator:
# Communicators built without __init__ (e.g. test doubles) publish no LoRA layout.
# Communicators built without __init__ (e.g. test doubles) publish no LoRA
# layout and try no fused kernel at the FFN exit.
_publish_lora_layout: bool = False
# Whether this class's boundary steps may be chosen from both sides'
# declarations; a subclass that picks its own steps says no.
_takes_declared_boundaries = True
_ffn_exit_fusions: Tuple[Callable, ...] = ()
# A plain residual unless the layer is built with its own.
_residual_ops: ResidualOps = ADD_AND_NORM
@@ -847,6 +860,8 @@ class LayerCommunicator:
)
# The fused kernels every batch's attention input tries first.
self._attn_input_fusions = self._select_attn_input_fusions()
# The fused kernels of the next layer's input the FFN exit tries first.
self._ffn_exit_fusions = self._select_ffn_exit_fusions()
self._speculative_algo = SpeculativeAlgorithm.from_string(
get_spec().speculative_algorithm
)
@@ -1004,8 +1019,7 @@ class LayerCommunicator:
)
if not (
self._takes_declared_boundaries
and (parallel.attn_cp_size == 1 or _cp_on_declarations())
(parallel.attn_cp_size == 1 or _cp_on_declarations())
# MoE layers under attention DP and GQA prefill CP keep the
# scatter-mode steps.
and not (
@@ -1544,9 +1558,14 @@ class LayerCommunicator:
mlp_reduce_scatter=mlp_reduce_scatter,
complete=complete_now,
)
defer_moe_finalize = self.should_defer_moe_finalize(forward_batch)
# Deferring implies fusing: a handoff skips the post-experts all-reduce.
fuse_mlp_allreduce = defer_moe_finalize or self._ffn_sum_moves_to_next_layer(
fusion = next(
filter(None, (fused(forward_batch) for fused in self._ffn_exit_fusions)),
None,
)
defer_moe_finalize = fusion is FfnExitFusion.DEFER_MOE_FINALIZE
# A fused kernel that takes the sum, a handoff included, skips the
# post-experts all-reduce.
fuse_mlp_allreduce = fusion is not None or self._ffn_sum_moves_to_next_layer(
forward_batch, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step
)
if fuse_mlp_allreduce:
@@ -1629,13 +1648,19 @@ class LayerCommunicator:
def finish_layer_stack(
self,
hidden_states: Union[torch.Tensor, UnreducedOutput],
hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput],
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
*,
final_norm_takes_handoff: bool = False,
) -> Tuple[Union[torch.Tensor, HandoffOutput], Optional[torch.Tensor]]:
"""Complete what this layer left for a next layer. Call it on the last
layer of this rank before its output reaches the final norm, the next
pipeline rank, or any other consumer outside the layers."""
pipeline rank, or any other consumer outside the layers. A final norm
that does a producer's handoff together with its own work
(``final_norm_takes_handoff``) receives it as it is."""
if final_norm_takes_handoff and isinstance(hidden_states, HandoffOutput):
return hidden_states, residual
return reduce_output(hidden_states), residual
def should_use_reduce_scatter(self, forward_batch: ForwardBatch):
@@ -1645,11 +1670,21 @@ class LayerCommunicator:
forward_batch, self._postprocess_dp_step(forward_batch)
)
def should_defer_moe_finalize(
self, forward_batch: ForwardBatch, m: int | None = None
) -> bool:
"""Whether the MoE may hand an unfinalized output to the next layer."""
return False
def _select_ffn_exit_fusions(
self,
) -> Tuple[Callable[[ForwardBatch], Optional["FfnExitFusion"]], ...]:
"""The fused kernels of the next layer's input that may take this layer's
FFN sum, in the order they are tried. Each returns what the exit does
when its kernel takes this batch, or None."""
return (self._next_input_norm_takes_ffn_sum,)
def _next_input_norm_takes_ffn_sum(
self, forward_batch: ForwardBatch
) -> Optional["FfnExitFusion"]:
"""The aiter / flashinfer AR + add + norm of the next layer's input."""
if self.should_fuse_mlp_allreduce_with_next_layer(forward_batch):
return FfnExitFusion.NEXT_INPUT
return None
def _ffn_sum_can_move_to_next_layer(self, forward_batch: ForwardBatch) -> bool:
# Under the MoE-CP all-gather the fusion path would skip postprocess_layer
@@ -1712,10 +1747,8 @@ class LayerCommunicator:
dp_step: Optional[Callable],
) -> bool:
"""Whether the FFN leaves its output's all-reduce to the next layer's
input norm: whenever the fused kernel takes it, and otherwise when the
next layer would run the same all-reduce the FFN itself would have."""
if self.should_fuse_mlp_allreduce_with_next_layer(forward_batch):
return True
input when no fused kernel takes it: when the next layer would run the
same all-reduce the FFN itself would have."""
return (
self._context.tp_size > 1
and not self.is_last_layer
@@ -1761,6 +1794,16 @@ def scatter_mode_layouts(
}
class FfnExitFusion(Enum):
"""What an FFN exit does when a fused kernel of the next layer's input takes
its sum."""
# The MoE hands its unfinalized output on; the next input finalizes it.
DEFER_MOE_FINALIZE = auto()
# The FFN leaves its all-reduce to the next input norm.
NEXT_INPUT = auto()
class FfnCompletion(msgspec.Struct, frozen=True):
"""One FFN's reduction decision. The flags are published while the FFN runs;
``complete(hidden_states, residual)`` then either wraps the output for the
@@ -2432,7 +2475,11 @@ def _attention_input_step(
result = fused(owed, residual, forward_batch, post_residual_addition)
if result is not None:
return result
hidden_states = owed.partial
if isinstance(owed, HandoffOutput):
# No fused kernel took the handoff: its producer completes it.
hidden_states, owed = reduce_output(owed), None
else:
hidden_states = owed.partial
if layer_input is not None:
hidden_states, residual = layer_input(hidden_states, residual, context)
if enters:
@@ -10,14 +10,14 @@ from __future__ import annotations
import logging
from typing import Callable, Optional, Sequence
import msgspec
import torch
from sglang.srt.layers.boundary_layout import SumGroup
from sglang.srt.layers.boundary_layout import SumGroup, TokenAxis
from sglang.srt.layers.communicator import (
FfnExitFusion,
FusedMlpInput,
HandoffOutput,
LayerCommunicator,
ScatterMode,
UnreducedOutput,
get_attn_tp_context,
)
@@ -74,14 +74,20 @@ def _resolve_max_m(*, max_running_requests: int | None) -> int:
return max(positive)
class MoeFinalizeHandoff(msgspec.Struct, frozen=True):
"""Unfinalized routed output plus the separately gated shared contribution."""
class MoeFinalizeHandoff(HandoffOutput, frozen=True):
"""Unfinalized routed output plus the separately gated shared contribution.
The next layer's fused finalize + AR + add + norm takes it; ``finish`` is
the MoE's own unfused tail, for any other reader."""
routed_output: torch.Tensor
expert_weights: torch.Tensor
permuted_indices: torch.Tensor
gated_shared_output: torch.Tensor
m: int
finish: Callable[[], torch.Tensor]
def complete(self) -> torch.Tensor:
return self.finish()
@classmethod
def from_flashinfer(
@@ -90,7 +96,13 @@ class MoeFinalizeHandoff(msgspec.Struct, frozen=True):
*,
gated_shared_output: torch.Tensor,
m: int,
reduce: Callable[[torch.Tensor], torch.Tensor],
) -> MoeFinalizeHandoff:
"""``reduce`` is the MoE's own sum of its finalized output."""
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
finalize_flashinfer_trtllm_deferred_output,
)
top_k = int(deferred_output.top_k)
return cls(
routed_output=deferred_output.gemm2_out.view(
@@ -102,6 +114,11 @@ class MoeFinalizeHandoff(msgspec.Struct, frozen=True):
)[:m],
gated_shared_output=gated_shared_output,
m=int(m),
finish=lambda: reduce(
finalize_flashinfer_trtllm_deferred_output(
deferred_output, gated_shared_output
)
),
)
@@ -178,17 +195,34 @@ class CuteDSLFusionService:
class CuteDSLFusionLayerCommunicator(LayerCommunicator):
# Chooses its own boundary steps, not from the declarations.
_takes_declared_boundaries = False
fusion_service: CuteDSLFusionService | None = None
# The FFN exit's CuteDSL kernels, which install() gives from what it knows
# of this layer and the one after it.
_cutedsl_exit_fusions: tuple = ()
# The runner can defer and a successor or the final norm consumes the handoff.
may_defer_moe_finalize: bool = False
# False on the last layer: the final norm does not all-reduce.
successor_absorbs_all_reduce: bool = False
# A replicated output follows the reduction; moving it would scale that by tp.
owes_local_reduction: bool = False
def install(
self,
service: CuteDSLFusionService,
*,
hands_off_finalize: bool,
next_input_absorbs: bool,
output_is_replicated: bool,
) -> None:
"""Take the model's fusion service and choose the FFN exit's CuteDSL
kernels: the MoE may hand off its finalize when it can and something
after it takes the handoff (``hands_off_finalize``); the FFN may leave
its all-reduce to the next layer's AR + norm when that layer has one,
unless a replicated output follows the reduction (moving it would scale
that by tp)."""
self.fusion_service = service
self._cutedsl_exit_fusions = (
(self._defer_moe_finalize_cutedsl,) if hands_off_finalize else ()
) + (
(self._absorb_all_reduce_cutedsl,)
if next_input_absorbs and not output_is_replicated
else ()
)
self._ffn_exit_fusions = self._select_ffn_exit_fusions()
def _select_attn_input_fusions(self):
return (
@@ -201,19 +235,13 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator):
self, owed, residual, forward_batch, post_residual_addition
):
"""Finish a deferred MoE finalize with the all-reduce, residual add and
input norm."""
input norm, when this layer's kernel takes the batch; otherwise the
handoff is completed by its producer's own tail."""
if not isinstance(owed, MoeFinalizeHandoff):
return None
if not self._should_use_finalize(forward_batch, owed.m):
raise RuntimeError(
"received deferred MoE output on an ineligible path "
f"(M={owed.m}, mode={forward_batch.forward_mode})"
)
gamma = _fused_norm_gamma(self.input_layernorm)
if gamma is None:
raise RuntimeError(
"deferred MoE finalize requires a fusable RMSNorm flavour"
)
if gamma is None or not self._should_use_finalize(forward_batch, owed.m):
return None
if post_residual_addition is not None:
residual = residual + post_residual_addition
assert self.fusion_service is not None
@@ -245,13 +273,14 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator):
fusions = super()._select_mlp_input_fusions()
parallel = get_parallel()
if (
self.layer_scatter_modes.layer_input_mode is ScatterMode.TP_ATTN_FULL
TokenAxis.ATTN_TP_SCATTER not in self.input_rows.sharded
# The workspace sums over TP, which is then the attention-TP group.
and parallel.attn_tp_size == parallel.tp_size
and _fused_norm_gamma(self.post_attention_layernorm) is not None
):
return (
FusedMlpInput(
completes=SumGroup.TP,
completes=SumGroup.ATTN_TP,
run=self._mlp_input_reduce_output_and_update_and_read_residual_cutedsl,
may_return_new_residual=True,
),
@@ -292,25 +321,29 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator):
and not get_exec().comm.enable_quant_communications
)
def _can_absorb_post_moe_all_reduce(
self, forward_batch: ForwardBatch, m: int
) -> bool:
"""Outgoing: skip our own all-reduce because the next layer absorbs it."""
return (
self.successor_absorbs_all_reduce
and not self.owes_local_reduction
and self._can_consume_post_moe_all_reduce(forward_batch, m)
)
def _select_ffn_exit_fusions(self):
return (*self._cutedsl_exit_fusions, *super()._select_ffn_exit_fusions())
def should_defer_moe_finalize(
self, forward_batch: ForwardBatch, m: int | None = None
) -> bool:
"""Deferring skips the post-experts all-reduce on the promise of a handoff."""
if not self.may_defer_moe_finalize:
return False
if m is None:
m = int(forward_batch.input_ids.shape[0])
return self._should_use_finalize(forward_batch, m)
def _defer_moe_finalize_cutedsl(
self, forward_batch: ForwardBatch
) -> Optional[FfnExitFusion]:
"""The MoE hands its unfinalized output to the next layer's finalize +
all-reduce + norm."""
if self._should_use_finalize(
forward_batch, int(forward_batch.input_ids.shape[0])
):
return FfnExitFusion.DEFER_MOE_FINALIZE
return None
def _absorb_all_reduce_cutedsl(
self, forward_batch: ForwardBatch
) -> Optional[FfnExitFusion]:
"""The next layer's AR + norm takes the post-experts all-reduce."""
if self._can_consume_post_moe_all_reduce(
forward_batch, int(forward_batch.input_ids.shape[0])
):
return FfnExitFusion.NEXT_INPUT
return None
def _common_eligible(self, forward_batch: ForwardBatch, m: int) -> bool:
parallel = get_parallel()
@@ -325,20 +358,11 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator):
and not get_attn_tp_context().input_scattered
and get_moe_a2a_backend().is_none()
and self._context.tp_size > 1
# Restates the base's moe-cp, MOE_FULL and SCATTERED refusals.
and self.layer_scatter_modes.mlp_mode is ScatterMode.FULL
# The FFN runs on the full rows, not each rank's own slice.
and TokenAxis.ATTN_TP_SCATTER
not in self._batch_steps(forward_batch).ffn_input_rows.sharded
)
def should_fuse_mlp_allreduce_with_next_layer(
self, forward_batch: ForwardBatch
) -> bool:
m = int(forward_batch.input_ids.shape[0])
if self.should_defer_moe_finalize(forward_batch, m):
return True
if self._can_absorb_post_moe_all_reduce(forward_batch, m):
return True
return super().should_fuse_mlp_allreduce_with_next_layer(forward_batch)
def install_cutedsl_fusion(
layers: Sequence[torch.nn.Module],
@@ -387,6 +411,7 @@ def install_cutedsl_fusion(
top_k=top_k,
rms_epsilon=rms_epsilon,
)
hands_off = 0
for index, layer in enumerate(layers):
communicator = layer.layer_communicator
if not isinstance(communicator, CuteDSLFusionLayerCommunicator):
@@ -398,15 +423,18 @@ def install_cutedsl_fusion(
has_consumer = isinstance(
successor.layer_communicator, CuteDSLFusionLayerCommunicator
)
communicator.fusion_service = service
communicator.may_defer_moe_finalize = (
bool(can_defer_finalize(layer)) and has_consumer
)
communicator.successor_absorbs_all_reduce = successor is not None and (
isinstance(successor.layer_communicator, CuteDSLFusionLayerCommunicator)
)
communicator.owes_local_reduction = (
requires_local_reduction is not None and requires_local_reduction(layer)
hands_off_finalize = bool(can_defer_finalize(layer)) and has_consumer
hands_off += hands_off_finalize
communicator.install(
service,
hands_off_finalize=hands_off_finalize,
# False on the last layer: the final norm does not all-reduce.
next_input_absorbs=successor is not None
and isinstance(
successor.layer_communicator, CuteDSLFusionLayerCommunicator
),
output_is_replicated=requires_local_reduction is not None
and bool(requires_local_reduction(layer)),
)
logger.info(
"Installed one %s FlashInfer MNNVL CuTe DSL fusion handle for %d of %d layers "
@@ -414,7 +442,7 @@ def install_cutedsl_fusion(
label,
len(fusion_layers),
len(layers),
sum(layer.layer_communicator.may_defer_moe_finalize for layer in fusion_layers),
hands_off,
)
return service
+1
View File
@@ -1113,6 +1113,7 @@ class DeepseekV2MoE(nn.Module):
final_hidden_states,
gated_shared_output=shared_output,
m=hidden_states.shape[0],
reduce=post_experts_all_reduce,
)
all_reduce_done = False
+1
View File
@@ -865,6 +865,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
final_hidden_states,
gated_shared_output=shared_output,
m=num_tokens,
reduce=reduce_moe_output,
)
if shared_output is not None:
+6 -1
View File
@@ -1856,7 +1856,12 @@ class Qwen3_5ForCausalLM(nn.Module):
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
hidden_states,
residual,
forward_batch,
# The final norm below finalizes a deferred MoE output in its kernel.
final_norm_takes_handoff=self.flashinfer_mnnvl_cutedsl_fusion is not None
and self.pp_group.is_last_rank,
)
# Return intermediate tensors for pipeline parallelism