[Refactor] Move CuTe DSL-fused layers onto the declared boundaries and FFN-exit kernel entries (#41431)
This commit is contained in:
@@ -761,14 +761,28 @@ class UnreducedOutput(msgspec.Struct, frozen=True):
|
||||
reduce_and_redistribute: Optional[Callable[[torch.Tensor], torch.Tensor]] = None
|
||||
|
||||
|
||||
class HandoffOutput(msgspec.Struct, frozen=True):
|
||||
"""A layer output that still owes work only its producer knows how to do (a
|
||||
MoE's finalize and sum), left for the next layer's input. A fused kernel
|
||||
there may do that work together with its own; anything else passes it
|
||||
through reduce_output(), which calls ``complete()``."""
|
||||
|
||||
def complete(self) -> torch.Tensor:
|
||||
"""Do the owed work, unfused, and return the complete output."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def reduce_output(
|
||||
hidden_states: Union[torch.Tensor, UnreducedOutput, None],
|
||||
hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput, None],
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Run the reduction an UnreducedOutput still owes; pass anything else through."""
|
||||
"""Run the work an UnreducedOutput or a HandoffOutput still owes; pass
|
||||
anything else through."""
|
||||
if isinstance(hidden_states, UnreducedOutput):
|
||||
if hidden_states.reduce_and_redistribute is not None:
|
||||
return hidden_states.reduce_and_redistribute(hidden_states.partial)
|
||||
return hidden_states.group.all_reduce(hidden_states.partial)
|
||||
if isinstance(hidden_states, HandoffOutput):
|
||||
return hidden_states.complete()
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -802,11 +816,10 @@ def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool:
|
||||
|
||||
|
||||
class LayerCommunicator:
|
||||
# Communicators built without __init__ (e.g. test doubles) publish no LoRA layout.
|
||||
# Communicators built without __init__ (e.g. test doubles) publish no LoRA
|
||||
# layout and try no fused kernel at the FFN exit.
|
||||
_publish_lora_layout: bool = False
|
||||
# Whether this class's boundary steps may be chosen from both sides'
|
||||
# declarations; a subclass that picks its own steps says no.
|
||||
_takes_declared_boundaries = True
|
||||
_ffn_exit_fusions: Tuple[Callable, ...] = ()
|
||||
# A plain residual unless the layer is built with its own.
|
||||
_residual_ops: ResidualOps = ADD_AND_NORM
|
||||
|
||||
@@ -847,6 +860,8 @@ class LayerCommunicator:
|
||||
)
|
||||
# The fused kernels every batch's attention input tries first.
|
||||
self._attn_input_fusions = self._select_attn_input_fusions()
|
||||
# The fused kernels of the next layer's input the FFN exit tries first.
|
||||
self._ffn_exit_fusions = self._select_ffn_exit_fusions()
|
||||
self._speculative_algo = SpeculativeAlgorithm.from_string(
|
||||
get_spec().speculative_algorithm
|
||||
)
|
||||
@@ -1004,8 +1019,7 @@ class LayerCommunicator:
|
||||
)
|
||||
|
||||
if not (
|
||||
self._takes_declared_boundaries
|
||||
and (parallel.attn_cp_size == 1 or _cp_on_declarations())
|
||||
(parallel.attn_cp_size == 1 or _cp_on_declarations())
|
||||
# MoE layers under attention DP and GQA prefill CP keep the
|
||||
# scatter-mode steps.
|
||||
and not (
|
||||
@@ -1544,9 +1558,14 @@ class LayerCommunicator:
|
||||
mlp_reduce_scatter=mlp_reduce_scatter,
|
||||
complete=complete_now,
|
||||
)
|
||||
defer_moe_finalize = self.should_defer_moe_finalize(forward_batch)
|
||||
# Deferring implies fusing: a handoff skips the post-experts all-reduce.
|
||||
fuse_mlp_allreduce = defer_moe_finalize or self._ffn_sum_moves_to_next_layer(
|
||||
fusion = next(
|
||||
filter(None, (fused(forward_batch) for fused in self._ffn_exit_fusions)),
|
||||
None,
|
||||
)
|
||||
defer_moe_finalize = fusion is FfnExitFusion.DEFER_MOE_FINALIZE
|
||||
# A fused kernel that takes the sum, a handoff included, skips the
|
||||
# post-experts all-reduce.
|
||||
fuse_mlp_allreduce = fusion is not None or self._ffn_sum_moves_to_next_layer(
|
||||
forward_batch, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step
|
||||
)
|
||||
if fuse_mlp_allreduce:
|
||||
@@ -1629,13 +1648,19 @@ class LayerCommunicator:
|
||||
|
||||
def finish_layer_stack(
|
||||
self,
|
||||
hidden_states: Union[torch.Tensor, UnreducedOutput],
|
||||
hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput],
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
*,
|
||||
final_norm_takes_handoff: bool = False,
|
||||
) -> Tuple[Union[torch.Tensor, HandoffOutput], Optional[torch.Tensor]]:
|
||||
"""Complete what this layer left for a next layer. Call it on the last
|
||||
layer of this rank before its output reaches the final norm, the next
|
||||
pipeline rank, or any other consumer outside the layers."""
|
||||
pipeline rank, or any other consumer outside the layers. A final norm
|
||||
that does a producer's handoff together with its own work
|
||||
(``final_norm_takes_handoff``) receives it as it is."""
|
||||
if final_norm_takes_handoff and isinstance(hidden_states, HandoffOutput):
|
||||
return hidden_states, residual
|
||||
return reduce_output(hidden_states), residual
|
||||
|
||||
def should_use_reduce_scatter(self, forward_batch: ForwardBatch):
|
||||
@@ -1645,11 +1670,21 @@ class LayerCommunicator:
|
||||
forward_batch, self._postprocess_dp_step(forward_batch)
|
||||
)
|
||||
|
||||
def should_defer_moe_finalize(
|
||||
self, forward_batch: ForwardBatch, m: int | None = None
|
||||
) -> bool:
|
||||
"""Whether the MoE may hand an unfinalized output to the next layer."""
|
||||
return False
|
||||
def _select_ffn_exit_fusions(
|
||||
self,
|
||||
) -> Tuple[Callable[[ForwardBatch], Optional["FfnExitFusion"]], ...]:
|
||||
"""The fused kernels of the next layer's input that may take this layer's
|
||||
FFN sum, in the order they are tried. Each returns what the exit does
|
||||
when its kernel takes this batch, or None."""
|
||||
return (self._next_input_norm_takes_ffn_sum,)
|
||||
|
||||
def _next_input_norm_takes_ffn_sum(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> Optional["FfnExitFusion"]:
|
||||
"""The aiter / flashinfer AR + add + norm of the next layer's input."""
|
||||
if self.should_fuse_mlp_allreduce_with_next_layer(forward_batch):
|
||||
return FfnExitFusion.NEXT_INPUT
|
||||
return None
|
||||
|
||||
def _ffn_sum_can_move_to_next_layer(self, forward_batch: ForwardBatch) -> bool:
|
||||
# Under the MoE-CP all-gather the fusion path would skip postprocess_layer
|
||||
@@ -1712,10 +1747,8 @@ class LayerCommunicator:
|
||||
dp_step: Optional[Callable],
|
||||
) -> bool:
|
||||
"""Whether the FFN leaves its output's all-reduce to the next layer's
|
||||
input norm: whenever the fused kernel takes it, and otherwise when the
|
||||
next layer would run the same all-reduce the FFN itself would have."""
|
||||
if self.should_fuse_mlp_allreduce_with_next_layer(forward_batch):
|
||||
return True
|
||||
input when no fused kernel takes it: when the next layer would run the
|
||||
same all-reduce the FFN itself would have."""
|
||||
return (
|
||||
self._context.tp_size > 1
|
||||
and not self.is_last_layer
|
||||
@@ -1761,6 +1794,16 @@ def scatter_mode_layouts(
|
||||
}
|
||||
|
||||
|
||||
class FfnExitFusion(Enum):
|
||||
"""What an FFN exit does when a fused kernel of the next layer's input takes
|
||||
its sum."""
|
||||
|
||||
# The MoE hands its unfinalized output on; the next input finalizes it.
|
||||
DEFER_MOE_FINALIZE = auto()
|
||||
# The FFN leaves its all-reduce to the next input norm.
|
||||
NEXT_INPUT = auto()
|
||||
|
||||
|
||||
class FfnCompletion(msgspec.Struct, frozen=True):
|
||||
"""One FFN's reduction decision. The flags are published while the FFN runs;
|
||||
``complete(hidden_states, residual)`` then either wraps the output for the
|
||||
@@ -2432,7 +2475,11 @@ def _attention_input_step(
|
||||
result = fused(owed, residual, forward_batch, post_residual_addition)
|
||||
if result is not None:
|
||||
return result
|
||||
hidden_states = owed.partial
|
||||
if isinstance(owed, HandoffOutput):
|
||||
# No fused kernel took the handoff: its producer completes it.
|
||||
hidden_states, owed = reduce_output(owed), None
|
||||
else:
|
||||
hidden_states = owed.partial
|
||||
if layer_input is not None:
|
||||
hidden_states, residual = layer_input(hidden_states, residual, context)
|
||||
if enters:
|
||||
|
||||
@@ -10,14 +10,14 @@ from __future__ import annotations
|
||||
import logging
|
||||
from typing import Callable, Optional, Sequence
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.boundary_layout import SumGroup
|
||||
from sglang.srt.layers.boundary_layout import SumGroup, TokenAxis
|
||||
from sglang.srt.layers.communicator import (
|
||||
FfnExitFusion,
|
||||
FusedMlpInput,
|
||||
HandoffOutput,
|
||||
LayerCommunicator,
|
||||
ScatterMode,
|
||||
UnreducedOutput,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
@@ -74,14 +74,20 @@ def _resolve_max_m(*, max_running_requests: int | None) -> int:
|
||||
return max(positive)
|
||||
|
||||
|
||||
class MoeFinalizeHandoff(msgspec.Struct, frozen=True):
|
||||
"""Unfinalized routed output plus the separately gated shared contribution."""
|
||||
class MoeFinalizeHandoff(HandoffOutput, frozen=True):
|
||||
"""Unfinalized routed output plus the separately gated shared contribution.
|
||||
The next layer's fused finalize + AR + add + norm takes it; ``finish`` is
|
||||
the MoE's own unfused tail, for any other reader."""
|
||||
|
||||
routed_output: torch.Tensor
|
||||
expert_weights: torch.Tensor
|
||||
permuted_indices: torch.Tensor
|
||||
gated_shared_output: torch.Tensor
|
||||
m: int
|
||||
finish: Callable[[], torch.Tensor]
|
||||
|
||||
def complete(self) -> torch.Tensor:
|
||||
return self.finish()
|
||||
|
||||
@classmethod
|
||||
def from_flashinfer(
|
||||
@@ -90,7 +96,13 @@ class MoeFinalizeHandoff(msgspec.Struct, frozen=True):
|
||||
*,
|
||||
gated_shared_output: torch.Tensor,
|
||||
m: int,
|
||||
reduce: Callable[[torch.Tensor], torch.Tensor],
|
||||
) -> MoeFinalizeHandoff:
|
||||
"""``reduce`` is the MoE's own sum of its finalized output."""
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
|
||||
finalize_flashinfer_trtllm_deferred_output,
|
||||
)
|
||||
|
||||
top_k = int(deferred_output.top_k)
|
||||
return cls(
|
||||
routed_output=deferred_output.gemm2_out.view(
|
||||
@@ -102,6 +114,11 @@ class MoeFinalizeHandoff(msgspec.Struct, frozen=True):
|
||||
)[:m],
|
||||
gated_shared_output=gated_shared_output,
|
||||
m=int(m),
|
||||
finish=lambda: reduce(
|
||||
finalize_flashinfer_trtllm_deferred_output(
|
||||
deferred_output, gated_shared_output
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -178,17 +195,34 @@ class CuteDSLFusionService:
|
||||
|
||||
|
||||
class CuteDSLFusionLayerCommunicator(LayerCommunicator):
|
||||
# Chooses its own boundary steps, not from the declarations.
|
||||
_takes_declared_boundaries = False
|
||||
|
||||
fusion_service: CuteDSLFusionService | None = None
|
||||
# The FFN exit's CuteDSL kernels, which install() gives from what it knows
|
||||
# of this layer and the one after it.
|
||||
_cutedsl_exit_fusions: tuple = ()
|
||||
|
||||
# The runner can defer and a successor or the final norm consumes the handoff.
|
||||
may_defer_moe_finalize: bool = False
|
||||
# False on the last layer: the final norm does not all-reduce.
|
||||
successor_absorbs_all_reduce: bool = False
|
||||
# A replicated output follows the reduction; moving it would scale that by tp.
|
||||
owes_local_reduction: bool = False
|
||||
def install(
|
||||
self,
|
||||
service: CuteDSLFusionService,
|
||||
*,
|
||||
hands_off_finalize: bool,
|
||||
next_input_absorbs: bool,
|
||||
output_is_replicated: bool,
|
||||
) -> None:
|
||||
"""Take the model's fusion service and choose the FFN exit's CuteDSL
|
||||
kernels: the MoE may hand off its finalize when it can and something
|
||||
after it takes the handoff (``hands_off_finalize``); the FFN may leave
|
||||
its all-reduce to the next layer's AR + norm when that layer has one,
|
||||
unless a replicated output follows the reduction (moving it would scale
|
||||
that by tp)."""
|
||||
self.fusion_service = service
|
||||
self._cutedsl_exit_fusions = (
|
||||
(self._defer_moe_finalize_cutedsl,) if hands_off_finalize else ()
|
||||
) + (
|
||||
(self._absorb_all_reduce_cutedsl,)
|
||||
if next_input_absorbs and not output_is_replicated
|
||||
else ()
|
||||
)
|
||||
self._ffn_exit_fusions = self._select_ffn_exit_fusions()
|
||||
|
||||
def _select_attn_input_fusions(self):
|
||||
return (
|
||||
@@ -201,19 +235,13 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator):
|
||||
self, owed, residual, forward_batch, post_residual_addition
|
||||
):
|
||||
"""Finish a deferred MoE finalize with the all-reduce, residual add and
|
||||
input norm."""
|
||||
input norm, when this layer's kernel takes the batch; otherwise the
|
||||
handoff is completed by its producer's own tail."""
|
||||
if not isinstance(owed, MoeFinalizeHandoff):
|
||||
return None
|
||||
if not self._should_use_finalize(forward_batch, owed.m):
|
||||
raise RuntimeError(
|
||||
"received deferred MoE output on an ineligible path "
|
||||
f"(M={owed.m}, mode={forward_batch.forward_mode})"
|
||||
)
|
||||
gamma = _fused_norm_gamma(self.input_layernorm)
|
||||
if gamma is None:
|
||||
raise RuntimeError(
|
||||
"deferred MoE finalize requires a fusable RMSNorm flavour"
|
||||
)
|
||||
if gamma is None or not self._should_use_finalize(forward_batch, owed.m):
|
||||
return None
|
||||
if post_residual_addition is not None:
|
||||
residual = residual + post_residual_addition
|
||||
assert self.fusion_service is not None
|
||||
@@ -245,13 +273,14 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator):
|
||||
fusions = super()._select_mlp_input_fusions()
|
||||
parallel = get_parallel()
|
||||
if (
|
||||
self.layer_scatter_modes.layer_input_mode is ScatterMode.TP_ATTN_FULL
|
||||
TokenAxis.ATTN_TP_SCATTER not in self.input_rows.sharded
|
||||
# The workspace sums over TP, which is then the attention-TP group.
|
||||
and parallel.attn_tp_size == parallel.tp_size
|
||||
and _fused_norm_gamma(self.post_attention_layernorm) is not None
|
||||
):
|
||||
return (
|
||||
FusedMlpInput(
|
||||
completes=SumGroup.TP,
|
||||
completes=SumGroup.ATTN_TP,
|
||||
run=self._mlp_input_reduce_output_and_update_and_read_residual_cutedsl,
|
||||
may_return_new_residual=True,
|
||||
),
|
||||
@@ -292,25 +321,29 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator):
|
||||
and not get_exec().comm.enable_quant_communications
|
||||
)
|
||||
|
||||
def _can_absorb_post_moe_all_reduce(
|
||||
self, forward_batch: ForwardBatch, m: int
|
||||
) -> bool:
|
||||
"""Outgoing: skip our own all-reduce because the next layer absorbs it."""
|
||||
return (
|
||||
self.successor_absorbs_all_reduce
|
||||
and not self.owes_local_reduction
|
||||
and self._can_consume_post_moe_all_reduce(forward_batch, m)
|
||||
)
|
||||
def _select_ffn_exit_fusions(self):
|
||||
return (*self._cutedsl_exit_fusions, *super()._select_ffn_exit_fusions())
|
||||
|
||||
def should_defer_moe_finalize(
|
||||
self, forward_batch: ForwardBatch, m: int | None = None
|
||||
) -> bool:
|
||||
"""Deferring skips the post-experts all-reduce on the promise of a handoff."""
|
||||
if not self.may_defer_moe_finalize:
|
||||
return False
|
||||
if m is None:
|
||||
m = int(forward_batch.input_ids.shape[0])
|
||||
return self._should_use_finalize(forward_batch, m)
|
||||
def _defer_moe_finalize_cutedsl(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> Optional[FfnExitFusion]:
|
||||
"""The MoE hands its unfinalized output to the next layer's finalize +
|
||||
all-reduce + norm."""
|
||||
if self._should_use_finalize(
|
||||
forward_batch, int(forward_batch.input_ids.shape[0])
|
||||
):
|
||||
return FfnExitFusion.DEFER_MOE_FINALIZE
|
||||
return None
|
||||
|
||||
def _absorb_all_reduce_cutedsl(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> Optional[FfnExitFusion]:
|
||||
"""The next layer's AR + norm takes the post-experts all-reduce."""
|
||||
if self._can_consume_post_moe_all_reduce(
|
||||
forward_batch, int(forward_batch.input_ids.shape[0])
|
||||
):
|
||||
return FfnExitFusion.NEXT_INPUT
|
||||
return None
|
||||
|
||||
def _common_eligible(self, forward_batch: ForwardBatch, m: int) -> bool:
|
||||
parallel = get_parallel()
|
||||
@@ -325,20 +358,11 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator):
|
||||
and not get_attn_tp_context().input_scattered
|
||||
and get_moe_a2a_backend().is_none()
|
||||
and self._context.tp_size > 1
|
||||
# Restates the base's moe-cp, MOE_FULL and SCATTERED refusals.
|
||||
and self.layer_scatter_modes.mlp_mode is ScatterMode.FULL
|
||||
# The FFN runs on the full rows, not each rank's own slice.
|
||||
and TokenAxis.ATTN_TP_SCATTER
|
||||
not in self._batch_steps(forward_batch).ffn_input_rows.sharded
|
||||
)
|
||||
|
||||
def should_fuse_mlp_allreduce_with_next_layer(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> bool:
|
||||
m = int(forward_batch.input_ids.shape[0])
|
||||
if self.should_defer_moe_finalize(forward_batch, m):
|
||||
return True
|
||||
if self._can_absorb_post_moe_all_reduce(forward_batch, m):
|
||||
return True
|
||||
return super().should_fuse_mlp_allreduce_with_next_layer(forward_batch)
|
||||
|
||||
|
||||
def install_cutedsl_fusion(
|
||||
layers: Sequence[torch.nn.Module],
|
||||
@@ -387,6 +411,7 @@ def install_cutedsl_fusion(
|
||||
top_k=top_k,
|
||||
rms_epsilon=rms_epsilon,
|
||||
)
|
||||
hands_off = 0
|
||||
for index, layer in enumerate(layers):
|
||||
communicator = layer.layer_communicator
|
||||
if not isinstance(communicator, CuteDSLFusionLayerCommunicator):
|
||||
@@ -398,15 +423,18 @@ def install_cutedsl_fusion(
|
||||
has_consumer = isinstance(
|
||||
successor.layer_communicator, CuteDSLFusionLayerCommunicator
|
||||
)
|
||||
communicator.fusion_service = service
|
||||
communicator.may_defer_moe_finalize = (
|
||||
bool(can_defer_finalize(layer)) and has_consumer
|
||||
)
|
||||
communicator.successor_absorbs_all_reduce = successor is not None and (
|
||||
isinstance(successor.layer_communicator, CuteDSLFusionLayerCommunicator)
|
||||
)
|
||||
communicator.owes_local_reduction = (
|
||||
requires_local_reduction is not None and requires_local_reduction(layer)
|
||||
hands_off_finalize = bool(can_defer_finalize(layer)) and has_consumer
|
||||
hands_off += hands_off_finalize
|
||||
communicator.install(
|
||||
service,
|
||||
hands_off_finalize=hands_off_finalize,
|
||||
# False on the last layer: the final norm does not all-reduce.
|
||||
next_input_absorbs=successor is not None
|
||||
and isinstance(
|
||||
successor.layer_communicator, CuteDSLFusionLayerCommunicator
|
||||
),
|
||||
output_is_replicated=requires_local_reduction is not None
|
||||
and bool(requires_local_reduction(layer)),
|
||||
)
|
||||
logger.info(
|
||||
"Installed one %s FlashInfer MNNVL CuTe DSL fusion handle for %d of %d layers "
|
||||
@@ -414,7 +442,7 @@ def install_cutedsl_fusion(
|
||||
label,
|
||||
len(fusion_layers),
|
||||
len(layers),
|
||||
sum(layer.layer_communicator.may_defer_moe_finalize for layer in fusion_layers),
|
||||
hands_off,
|
||||
)
|
||||
return service
|
||||
|
||||
|
||||
@@ -1113,6 +1113,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
final_hidden_states,
|
||||
gated_shared_output=shared_output,
|
||||
m=hidden_states.shape[0],
|
||||
reduce=post_experts_all_reduce,
|
||||
)
|
||||
|
||||
all_reduce_done = False
|
||||
|
||||
@@ -865,6 +865,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
final_hidden_states,
|
||||
gated_shared_output=shared_output,
|
||||
m=num_tokens,
|
||||
reduce=reduce_moe_output,
|
||||
)
|
||||
|
||||
if shared_output is not None:
|
||||
|
||||
@@ -1856,7 +1856,12 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
# The final norm below finalizes a deferred MoE output in its kernel.
|
||||
final_norm_takes_handoff=self.flashinfer_mnnvl_cutedsl_fusion is not None
|
||||
and self.pp_group.is_last_rank,
|
||||
)
|
||||
|
||||
# Return intermediate tensors for pipeline parallelism
|
||||
|
||||
Reference in New Issue
Block a user