[Refactor] Give a layer its CuTe DSL kernels at construction instead of a subclass (#41442)
This commit is contained in:
@@ -16,7 +16,7 @@
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from functools import cached_property, partial
|
||||
from typing import Callable, Optional, Tuple, Union
|
||||
from typing import Callable, Optional, Protocol, Tuple, Union
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
@@ -301,6 +301,22 @@ class StageCommunicator:
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
class LayerFusions(Protocol):
|
||||
"""The fused kernels a fusion backend gives a layer, bound to it and tried
|
||||
before the layer's own: at its attention input (each takes what the previous
|
||||
layer left, the residual, the batch and a post-residual addition), at its
|
||||
FFN input, and at its FFN exit (each says what the exit does when its kernel
|
||||
takes the batch)."""
|
||||
|
||||
def attention_input(self, layer: "LayerCommunicator") -> Tuple[Callable, ...]: ...
|
||||
|
||||
def ffn_input(self, layer: "LayerCommunicator") -> Tuple["FusedMlpInput", ...]: ...
|
||||
|
||||
def ffn_exit(
|
||||
self, layer: "LayerCommunicator"
|
||||
) -> Tuple[Callable[[ForwardBatch], Optional["FfnExitFusion"]], ...]: ...
|
||||
|
||||
|
||||
class LayerCommunicator:
|
||||
# Communicators built without __init__ (e.g. test doubles) publish no LoRA
|
||||
# layout and try no fused kernel at the FFN exit.
|
||||
@@ -308,6 +324,8 @@ class LayerCommunicator:
|
||||
_ffn_exit_fusions: Tuple[Callable, ...] = ()
|
||||
# A plain residual unless the layer is built with its own.
|
||||
_residual: LayerResidual = PLAIN_RESIDUAL
|
||||
# The fused kernels a backend gives the layer, if any.
|
||||
fusions: Optional[LayerFusions] = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -328,6 +346,8 @@ class LayerCommunicator:
|
||||
# A layer that is one stage of a sequence of stages, instead of an
|
||||
# attention followed by an FFN.
|
||||
stage: Optional["LayerStage"] = None,
|
||||
# The fused kernels a backend gives the layer, tried before its own.
|
||||
fusions: Optional[LayerFusions] = None,
|
||||
):
|
||||
self.layer_scatter_modes = layer_scatter_modes
|
||||
self.input_layernorm = input_layernorm
|
||||
@@ -340,6 +360,7 @@ class LayerCommunicator:
|
||||
self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16
|
||||
self.allow_deferred_ffn_reduction = allow_deferred_ffn_reduction
|
||||
self._residual = residual
|
||||
self.fusions = fusions
|
||||
|
||||
self._context = CommunicateContext.init_new()
|
||||
self._context.force_layernorm_before_dp_gather = (
|
||||
@@ -765,9 +786,10 @@ class LayerCommunicator:
|
||||
Each takes (owed, residual, forward_batch, post_residual_addition) and
|
||||
returns None when it does not take the batch. They add the residual
|
||||
plainly; the boundary tries them only when the update it writes in is a
|
||||
plain add."""
|
||||
plain add. A backend's come first."""
|
||||
given = self.fusions.attention_input(self) if self.fusions else ()
|
||||
if not hasattr(self.input_layernorm, "forward_with_allreduce_fusion"):
|
||||
return ()
|
||||
return given
|
||||
self._attn_input_fuses_quant = (
|
||||
self.enable_fused_ar_quant
|
||||
and _use_aiter
|
||||
@@ -775,7 +797,7 @@ class LayerCommunicator:
|
||||
self.input_layernorm, "forward_with_allreduce_fusion_quant_per_group"
|
||||
)
|
||||
)
|
||||
return (self._reduce_output_and_update_and_read_residual,)
|
||||
return (*given, self._reduce_output_and_update_and_read_residual)
|
||||
|
||||
def _reduce_output_and_update_and_read_residual(
|
||||
self,
|
||||
@@ -822,10 +844,13 @@ class LayerCommunicator:
|
||||
def _select_mlp_input_fusions(self) -> Tuple["FusedMlpInput", ...]:
|
||||
"""The fused kernels that can take the attention -> FFN steps, in the
|
||||
order they are tried. They add the residual plainly; the boundary tries
|
||||
them only when the update it writes in is a plain add."""
|
||||
them only when the update it writes in is a plain add. A backend's come
|
||||
first."""
|
||||
given = self.fusions.ffn_input(self) if self.fusions else ()
|
||||
if not hasattr(self.post_attention_layernorm, "forward_with_allreduce_fusion"):
|
||||
return ()
|
||||
return given
|
||||
return (
|
||||
*given,
|
||||
FusedMlpInput(
|
||||
completes=SumGroup.ATTN_TP,
|
||||
run=self._mlp_input_reduce_output_and_update_and_read_residual,
|
||||
@@ -1123,8 +1148,9 @@ class LayerCommunicator:
|
||||
) -> Tuple[Callable[[ForwardBatch], Optional["FfnExitFusion"]], ...]:
|
||||
"""The fused kernels of the next layer's input that may take this layer's
|
||||
FFN sum, in the order they are tried. Each returns what the exit does
|
||||
when its kernel takes this batch, or None."""
|
||||
return (self._next_input_norm_takes_ffn_sum,)
|
||||
when its kernel takes this batch, or None. A backend's come first."""
|
||||
given = self.fusions.ffn_exit(self) if self.fusions else ()
|
||||
return (*given, self._next_input_norm_takes_ffn_sum)
|
||||
|
||||
def _next_input_norm_takes_ffn_sum(
|
||||
self, forward_batch: ForwardBatch
|
||||
|
||||
@@ -8,6 +8,7 @@ shared-expert add folded in when the runner hands back a MoeFinalizeHandoff.
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from functools import partial
|
||||
from typing import Callable, Optional, Sequence
|
||||
|
||||
import torch
|
||||
@@ -195,11 +196,20 @@ class CuteDSLFusionService:
|
||||
)
|
||||
|
||||
|
||||
class CuteDSLFusionLayerCommunicator(LayerCommunicator):
|
||||
fusion_service: CuteDSLFusionService | None = None
|
||||
# The FFN exit's CuteDSL kernels, which install() gives from what it knows
|
||||
# of this layer and the one after it.
|
||||
_cutedsl_exit_fusions: tuple = ()
|
||||
class CuteDSLFusion:
|
||||
"""One layer's CuTe DSL kernels, given to its LayerCommunicator at
|
||||
construction. At the attention input: the MoE finalize + AR + add + norm of
|
||||
a handoff the previous layer left, and the AR + add + norm of a sum it left.
|
||||
At the FFN input: the attention output's AR + add + norm. At the FFN exit:
|
||||
handing the MoE finalize, or the all-reduce, to the next layer's kernel.
|
||||
install_cutedsl_fusion gives it the model's service and chooses the exit
|
||||
from the layer after this one; until then no kernel takes a batch."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.service: CuteDSLFusionService | None = None
|
||||
# The FFN exit's choices, from what install() knows of the next layer.
|
||||
self.hands_off_finalize = False
|
||||
self.next_input_absorbs = False
|
||||
|
||||
def install(
|
||||
self,
|
||||
@@ -209,162 +219,163 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator):
|
||||
next_input_absorbs: bool,
|
||||
output_is_replicated: bool,
|
||||
) -> None:
|
||||
"""Take the model's fusion service and choose the FFN exit's CuteDSL
|
||||
kernels: the MoE may hand off its finalize when it can and something
|
||||
after it takes the handoff (``hands_off_finalize``); the FFN may leave
|
||||
its all-reduce to the next layer's AR + norm when that layer has one,
|
||||
unless a replicated output follows the reduction (moving it would scale
|
||||
that by tp)."""
|
||||
self.fusion_service = service
|
||||
self._cutedsl_exit_fusions = (
|
||||
(self._defer_moe_finalize_cutedsl,) if hands_off_finalize else ()
|
||||
) + (
|
||||
(self._absorb_all_reduce_cutedsl,)
|
||||
if next_input_absorbs and not output_is_replicated
|
||||
else ()
|
||||
)
|
||||
self._ffn_exit_fusions = self._select_ffn_exit_fusions()
|
||||
"""Take the model's fusion service and choose the FFN exit's kernels:
|
||||
the MoE may hand off its finalize when it can and something after it
|
||||
takes the handoff (``hands_off_finalize``); the FFN may leave its
|
||||
all-reduce to the next layer's AR + norm when that layer has one, unless
|
||||
a replicated output follows the reduction (moving it would scale that
|
||||
by tp)."""
|
||||
self.service = service
|
||||
self.hands_off_finalize = hands_off_finalize
|
||||
self.next_input_absorbs = next_input_absorbs and not output_is_replicated
|
||||
|
||||
def _select_attn_input_fusions(self):
|
||||
def attention_input(self, layer: LayerCommunicator) -> tuple:
|
||||
return (
|
||||
self._finalize_output_and_update_and_read_residual_cutedsl,
|
||||
self._reduce_output_and_update_and_read_residual_cutedsl,
|
||||
*super()._select_attn_input_fusions(),
|
||||
partial(self._finalize_output_and_update_and_read_residual, layer),
|
||||
partial(self._reduce_output_and_update_and_read_residual, layer),
|
||||
)
|
||||
|
||||
def _finalize_output_and_update_and_read_residual_cutedsl(
|
||||
self, owed, residual, forward_batch, post_residual_addition
|
||||
def ffn_input(self, layer: LayerCommunicator) -> tuple:
|
||||
parallel = get_parallel()
|
||||
if (
|
||||
TokenAxis.ATTN_TP_SCATTER not in layer.input_rows.sharded
|
||||
# The workspace sums over TP, which is then the attention-TP group.
|
||||
and parallel.attn_tp_size == parallel.tp_size
|
||||
and _fused_norm_gamma(layer.post_attention_layernorm) is not None
|
||||
):
|
||||
return (
|
||||
FusedMlpInput(
|
||||
completes=SumGroup.ATTN_TP,
|
||||
run=partial(
|
||||
self._mlp_input_reduce_output_and_update_and_read_residual,
|
||||
layer,
|
||||
),
|
||||
may_return_new_residual=True,
|
||||
),
|
||||
)
|
||||
return ()
|
||||
|
||||
def ffn_exit(self, layer: LayerCommunicator) -> tuple:
|
||||
return (
|
||||
partial(self._defer_moe_finalize, layer),
|
||||
partial(self._absorb_all_reduce, layer),
|
||||
)
|
||||
|
||||
def _finalize_output_and_update_and_read_residual(
|
||||
self, layer, owed, residual, forward_batch, post_residual_addition
|
||||
):
|
||||
"""Finish a deferred MoE finalize with the all-reduce, residual add and
|
||||
input norm, when this layer's kernel takes the batch; otherwise the
|
||||
handoff is completed by its producer's own tail."""
|
||||
if not isinstance(owed, MoeFinalizeHandoff):
|
||||
return None
|
||||
gamma = _fused_norm_gamma(self.input_layernorm)
|
||||
if gamma is None or not self._should_use_finalize(forward_batch, owed.m):
|
||||
gamma = _fused_norm_gamma(layer.input_layernorm)
|
||||
if gamma is None or not self._should_use_finalize(layer, forward_batch, owed.m):
|
||||
return None
|
||||
if post_residual_addition is not None:
|
||||
residual = residual + post_residual_addition
|
||||
assert self.fusion_service is not None
|
||||
return self.fusion_service.finalize(
|
||||
handoff=owed, residual=residual, gamma=gamma
|
||||
)
|
||||
assert self.service is not None
|
||||
return self.service.finalize(handoff=owed, residual=residual, gamma=gamma)
|
||||
|
||||
def _reduce_output_and_update_and_read_residual_cutedsl(
|
||||
self, owed, residual, forward_batch, post_residual_addition
|
||||
def _reduce_output_and_update_and_read_residual(
|
||||
self, layer, owed, residual, forward_batch, post_residual_addition
|
||||
):
|
||||
"""Complete the all-reduce the previous layer left with the residual add
|
||||
and input norm."""
|
||||
if not isinstance(owed, UnreducedOutput) or not (
|
||||
self._can_consume_post_moe_all_reduce(
|
||||
forward_batch, int(owed.partial.shape[0])
|
||||
layer, forward_batch, int(owed.partial.shape[0])
|
||||
)
|
||||
):
|
||||
return None
|
||||
if post_residual_addition is not None:
|
||||
residual = residual + post_residual_addition
|
||||
assert self.fusion_service is not None
|
||||
return self.fusion_service.all_reduce_residual_rms_norm(
|
||||
assert self.service is not None
|
||||
return self.service.all_reduce_residual_rms_norm(
|
||||
local_contribution=owed.partial,
|
||||
residual=residual,
|
||||
gamma=_fused_norm_gamma(self.input_layernorm),
|
||||
gamma=_fused_norm_gamma(layer.input_layernorm),
|
||||
)
|
||||
|
||||
def _select_mlp_input_fusions(self):
|
||||
fusions = super()._select_mlp_input_fusions()
|
||||
parallel = get_parallel()
|
||||
if (
|
||||
TokenAxis.ATTN_TP_SCATTER not in self.input_rows.sharded
|
||||
# The workspace sums over TP, which is then the attention-TP group.
|
||||
and parallel.attn_tp_size == parallel.tp_size
|
||||
and _fused_norm_gamma(self.post_attention_layernorm) is not None
|
||||
):
|
||||
return (
|
||||
FusedMlpInput(
|
||||
completes=SumGroup.ATTN_TP,
|
||||
run=self._mlp_input_reduce_output_and_update_and_read_residual_cutedsl,
|
||||
may_return_new_residual=True,
|
||||
),
|
||||
*fusions,
|
||||
)
|
||||
return fusions
|
||||
|
||||
def _mlp_input_reduce_output_and_update_and_read_residual_cutedsl(
|
||||
self, hidden_states, residual, forward_batch
|
||||
def _mlp_input_reduce_output_and_update_and_read_residual(
|
||||
self, layer, hidden_states, residual, forward_batch
|
||||
):
|
||||
"""The attention output's all-reduce with the residual add and the
|
||||
post-attention norm."""
|
||||
if not (
|
||||
self._common_eligible(forward_batch, int(hidden_states.shape[0]))
|
||||
self._common_eligible(layer, forward_batch, int(hidden_states.shape[0]))
|
||||
and residual is not None
|
||||
and not get_exec().comm.enable_quant_communications
|
||||
):
|
||||
return None
|
||||
assert self.fusion_service is not None
|
||||
return self.fusion_service.all_reduce_residual_rms_norm(
|
||||
assert self.service is not None
|
||||
return self.service.all_reduce_residual_rms_norm(
|
||||
local_contribution=hidden_states,
|
||||
residual=residual,
|
||||
gamma=_fused_norm_gamma(self.post_attention_layernorm),
|
||||
gamma=_fused_norm_gamma(layer.post_attention_layernorm),
|
||||
)
|
||||
|
||||
def _should_use_finalize(self, forward_batch: ForwardBatch, m: int) -> bool:
|
||||
def _should_use_finalize(self, layer, forward_batch: ForwardBatch, m: int) -> bool:
|
||||
return (
|
||||
self._common_eligible(forward_batch, m) and get_parallel().moe_ep_size == 1
|
||||
self._common_eligible(layer, forward_batch, m)
|
||||
and get_parallel().moe_ep_size == 1
|
||||
)
|
||||
|
||||
def _can_consume_post_moe_all_reduce(
|
||||
self, forward_batch: ForwardBatch, m: int
|
||||
self, layer, forward_batch: ForwardBatch, m: int
|
||||
) -> bool:
|
||||
"""Incoming, and independent of this layer's own successor."""
|
||||
return (
|
||||
self._common_eligible(forward_batch, m)
|
||||
and _fused_norm_gamma(self.input_layernorm) is not None
|
||||
self._common_eligible(layer, forward_batch, m)
|
||||
and _fused_norm_gamma(layer.input_layernorm) is not None
|
||||
and not get_exec().comm.enable_quant_communications
|
||||
)
|
||||
|
||||
def _select_ffn_exit_fusions(self):
|
||||
return (*self._cutedsl_exit_fusions, *super()._select_ffn_exit_fusions())
|
||||
|
||||
def _defer_moe_finalize_cutedsl(
|
||||
self, forward_batch: ForwardBatch
|
||||
def _defer_moe_finalize(
|
||||
self, layer, forward_batch: ForwardBatch
|
||||
) -> Optional[FfnExitFusion]:
|
||||
"""The MoE hands its unfinalized output to the next layer's finalize +
|
||||
all-reduce + norm."""
|
||||
if self._should_use_finalize(
|
||||
forward_batch, int(forward_batch.input_ids.shape[0])
|
||||
if self.hands_off_finalize and self._should_use_finalize(
|
||||
layer, forward_batch, int(forward_batch.input_ids.shape[0])
|
||||
):
|
||||
return FfnExitFusion.DEFER_MOE_FINALIZE
|
||||
return None
|
||||
|
||||
def _absorb_all_reduce_cutedsl(
|
||||
self, forward_batch: ForwardBatch
|
||||
def _absorb_all_reduce(
|
||||
self, layer, forward_batch: ForwardBatch
|
||||
) -> Optional[FfnExitFusion]:
|
||||
"""The next layer's AR + norm takes the post-experts all-reduce."""
|
||||
if self._can_consume_post_moe_all_reduce(
|
||||
forward_batch, int(forward_batch.input_ids.shape[0])
|
||||
if self.next_input_absorbs and self._can_consume_post_moe_all_reduce(
|
||||
layer, forward_batch, int(forward_batch.input_ids.shape[0])
|
||||
):
|
||||
return FfnExitFusion.NEXT_INPUT
|
||||
return None
|
||||
|
||||
def _common_eligible(self, forward_batch: ForwardBatch, m: int) -> bool:
|
||||
def _common_eligible(self, layer, forward_batch: ForwardBatch, m: int) -> bool:
|
||||
parallel = get_parallel()
|
||||
return bool(
|
||||
self.fusion_service is not None
|
||||
self.service is not None
|
||||
and _is_supported_forward_mode(forward_batch.forward_mode)
|
||||
and self.fusion_service.supports(m)
|
||||
and self.service.supports(m)
|
||||
and not is_dp_attention_enabled()
|
||||
# Also forces moe_dp_size == 1, so a hybrid EP x MoE-TP reduction
|
||||
# always merges into the one TP reduction the workspace performs.
|
||||
and parallel.attn_cp_size == 1
|
||||
and not get_attn_tp_context().input_scattered
|
||||
and get_moe_a2a_backend().is_none()
|
||||
and self._context.tp_size > 1
|
||||
and parallel.tp_size > 1
|
||||
# The FFN runs on the full rows, not each rank's own slice.
|
||||
and TokenAxis.ATTN_TP_SCATTER
|
||||
not in self._batch_steps(forward_batch).ffn.input_rows.sharded
|
||||
not in layer.ffn.entry(forward_batch).input_rows.sharded
|
||||
)
|
||||
|
||||
|
||||
def _fusion_of(layer: torch.nn.Module) -> CuteDSLFusion | None:
|
||||
fusions = getattr(layer.layer_communicator, "fusions", None)
|
||||
return fusions if isinstance(fusions, CuteDSLFusion) else None
|
||||
|
||||
|
||||
def install_cutedsl_fusion(
|
||||
layers: Sequence[torch.nn.Module],
|
||||
*,
|
||||
@@ -384,11 +395,7 @@ def install_cutedsl_fusion(
|
||||
# A draft shares the target's process, which holds one workspace.
|
||||
return None
|
||||
|
||||
fusion_layers = [
|
||||
layer
|
||||
for layer in layers
|
||||
if isinstance(layer.layer_communicator, CuteDSLFusionLayerCommunicator)
|
||||
]
|
||||
fusion_layers = [layer for layer in layers if _fusion_of(layer) is not None]
|
||||
if not fusion_layers:
|
||||
return None
|
||||
|
||||
@@ -414,26 +421,22 @@ def install_cutedsl_fusion(
|
||||
)
|
||||
hands_off = 0
|
||||
for index, layer in enumerate(layers):
|
||||
communicator = layer.layer_communicator
|
||||
if not isinstance(communicator, CuteDSLFusionLayerCommunicator):
|
||||
fusion = _fusion_of(layer)
|
||||
if fusion is None:
|
||||
continue
|
||||
successor = layers[index + 1] if index + 1 < len(layers) else None
|
||||
if successor is None:
|
||||
has_consumer = final_norm_consumes_handoff
|
||||
else:
|
||||
has_consumer = isinstance(
|
||||
successor.layer_communicator, CuteDSLFusionLayerCommunicator
|
||||
)
|
||||
has_consumer = _fusion_of(successor) is not None
|
||||
hands_off_finalize = bool(can_defer_finalize(layer)) and has_consumer
|
||||
hands_off += hands_off_finalize
|
||||
communicator.install(
|
||||
fusion.install(
|
||||
service,
|
||||
hands_off_finalize=hands_off_finalize,
|
||||
# False on the last layer: the final norm does not all-reduce.
|
||||
next_input_absorbs=successor is not None
|
||||
and isinstance(
|
||||
successor.layer_communicator, CuteDSLFusionLayerCommunicator
|
||||
),
|
||||
and _fusion_of(successor) is not None,
|
||||
output_is_replicated=requires_local_reduction is not None
|
||||
and bool(requires_local_reduction(layer)),
|
||||
)
|
||||
@@ -456,16 +459,18 @@ def prepare_cutedsl_fusion(
|
||||
Scans the module tree, so a wrapper holding the model as a submodule (a VLM's
|
||||
language_model) needs no hook of its own.
|
||||
"""
|
||||
communicators = [
|
||||
communicator
|
||||
fusions = [
|
||||
fusion
|
||||
for module in model.modules()
|
||||
# Most modules carry no layer_communicator.
|
||||
if isinstance(
|
||||
communicator := module.__dict__.get("layer_communicator"),
|
||||
CuteDSLFusionLayerCommunicator,
|
||||
fusion := getattr(
|
||||
module.__dict__.get("layer_communicator"), "fusions", None
|
||||
),
|
||||
CuteDSLFusion,
|
||||
)
|
||||
]
|
||||
if not communicators or any(c.fusion_service is None for c in communicators):
|
||||
if not fusions or any(f.service is None for f in fusions):
|
||||
raise ValueError(
|
||||
"--flashinfer-allreduce-fusion-backend cutedsl is set, but "
|
||||
f"{type(model).__name__} installed no CuTe DSL fusion service, so no "
|
||||
@@ -477,7 +482,7 @@ def prepare_cutedsl_fusion(
|
||||
"FlashInfer MNNVL CuTe DSL fusion does not support concurrent PDMux "
|
||||
"streams sharing one mutable workspace"
|
||||
)
|
||||
services = {id(c.fusion_service): c.fusion_service for c in communicators}
|
||||
services = {id(f.service): f.service for f in fusions}
|
||||
max_m = _resolve_max_m(max_running_requests=max_running_requests)
|
||||
for service in services.values():
|
||||
service.prepare(max_m=max_m)
|
||||
@@ -485,5 +490,5 @@ def prepare_cutedsl_fusion(
|
||||
"Prepared the FlashInfer MNNVL CuTe DSL fusion workspace for M_max=%d "
|
||||
"(%d fused layers)",
|
||||
max_m,
|
||||
len(communicators),
|
||||
len(fusions),
|
||||
)
|
||||
|
||||
@@ -2632,26 +2632,24 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
):
|
||||
"""The communicator for this layer's norms; it chooses its boundary
|
||||
steps from them at construction."""
|
||||
fusions = None
|
||||
if (
|
||||
not get_parallel().enable_prefill_cp
|
||||
and not self.is_nextn
|
||||
and _use_mnnvl_cutedsl_fusion()
|
||||
):
|
||||
# Dense layers too: selecting cutedsl turns the legacy fusion off.
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import (
|
||||
CuteDSLFusionLayerCommunicator,
|
||||
)
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import CuteDSLFusion
|
||||
|
||||
communicator_cls = CuteDSLFusionLayerCommunicator
|
||||
else:
|
||||
communicator_cls = LayerCommunicator
|
||||
return communicator_cls(
|
||||
fusions = CuteDSLFusion()
|
||||
return LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
input_layernorm=input_layernorm,
|
||||
post_attention_layernorm=post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
qkv_latent_func=qkv_latent_func,
|
||||
allow_deferred_ffn_reduction=allow_deferred_ffn_reduction,
|
||||
fusions=fusions,
|
||||
)
|
||||
|
||||
def _detect_gfx95_quant_format(self) -> str:
|
||||
|
||||
@@ -214,14 +214,13 @@ def _use_mnnvl_cutedsl_fusion(config: Qwen3_5TextConfig, is_nextn: bool) -> bool
|
||||
)
|
||||
|
||||
|
||||
def _layer_communicator_class(config: Qwen3_5TextConfig, is_nextn: bool):
|
||||
def _layer_fusions(config: Qwen3_5TextConfig, is_nextn: bool):
|
||||
"""The CuTe DSL kernels a layer tries before its own, when they are on."""
|
||||
if _use_mnnvl_cutedsl_fusion(config, is_nextn):
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import (
|
||||
CuteDSLFusionLayerCommunicator,
|
||||
)
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import CuteDSLFusion
|
||||
|
||||
return CuteDSLFusionLayerCommunicator
|
||||
return LayerCommunicator
|
||||
return CuteDSLFusion()
|
||||
return None
|
||||
|
||||
|
||||
if _is_cuda:
|
||||
@@ -1124,13 +1123,14 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
_enable_qwen35_fused_ar_quant()
|
||||
and _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
|
||||
)
|
||||
self.layer_communicator = _layer_communicator_class(config, is_nextn)(
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
enable_fused_ar_quant=enable_fused_ar_quant,
|
||||
fused_ar_quant_keep_bf16=enable_fused_ar_quant,
|
||||
fusions=_layer_fusions(config, is_nextn),
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -1327,13 +1327,14 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
enable_fused_ar_quant = (
|
||||
_enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.qkv_proj)
|
||||
)
|
||||
self.layer_communicator = _layer_communicator_class(config, is_nextn)(
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
enable_fused_ar_quant=enable_fused_ar_quant,
|
||||
fused_ar_quant_keep_bf16=False,
|
||||
fusions=_layer_fusions(config, is_nextn),
|
||||
)
|
||||
|
||||
self.alt_stream = alt_stream
|
||||
|
||||
@@ -24,7 +24,7 @@ from sglang.srt.layers.flashinfer_mnnvl_cutedsl import (
|
||||
)
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import (
|
||||
CuteDSLFusionLayerCommunicator,
|
||||
CuteDSLFusion,
|
||||
MoeFinalizeHandoff,
|
||||
install_cutedsl_fusion,
|
||||
prepare_cutedsl_fusion,
|
||||
@@ -43,11 +43,13 @@ _DECODE = SimpleNamespace(forward_mode=ForwardMode.DECODE, input_ids=torch.zeros
|
||||
|
||||
|
||||
def _communicator():
|
||||
comm = CuteDSLFusionLayerCommunicator.__new__(CuteDSLFusionLayerCommunicator)
|
||||
comm = LayerCommunicator.__new__(LayerCommunicator)
|
||||
comm.fusions = CuteDSLFusion()
|
||||
comm.input_layernorm = RMSNorm(8, eps=1e-6)
|
||||
comm.post_attention_layernorm = RMSNorm(8, eps=1e-6)
|
||||
comm.enable_fused_ar_quant = False
|
||||
comm._attn_input_fusions = comm._select_attn_input_fusions()
|
||||
comm._ffn_exit_fusions = comm._select_ffn_exit_fusions()
|
||||
# Only the ordinary batches' attention input half, with these entries.
|
||||
comm._context = SimpleNamespace()
|
||||
comm._sp_steps = comm._input_scattered_steps = comm._cp_steps = None
|
||||
@@ -71,6 +73,11 @@ def _communicator():
|
||||
return comm
|
||||
|
||||
|
||||
def _bound(entry):
|
||||
"""What an entry runs and the layer it is bound to."""
|
||||
return entry.func, entry.args
|
||||
|
||||
|
||||
def _install(layers, **kwargs):
|
||||
reset_context()
|
||||
publish(ServerArgs(model_path="dummy"), role="test")
|
||||
@@ -89,9 +96,7 @@ def _install(layers, **kwargs):
|
||||
def eligible():
|
||||
"""Every shared gate open, so a case varies only the predicate it names."""
|
||||
with (
|
||||
patch.object(
|
||||
CuteDSLFusionLayerCommunicator, "_common_eligible", return_value=True
|
||||
),
|
||||
patch.object(CuteDSLFusion, "_common_eligible", return_value=True),
|
||||
patch(
|
||||
f"{_MODULE}.get_exec",
|
||||
return_value=SimpleNamespace(
|
||||
@@ -106,7 +111,7 @@ def test_last_layer_consumes_but_does_not_skip_the_pending_all_reduce(eligible):
|
||||
"""The last layer must fuse the reduction its predecessor skipped, or the
|
||||
unreduced output reaches the final norm; it must not skip its own."""
|
||||
last = _communicator()
|
||||
last.install(
|
||||
last.fusions.install(
|
||||
SimpleNamespace(
|
||||
all_reduce_residual_rms_norm=lambda *, local_contribution, residual, gamma: (
|
||||
local_contribution + 1,
|
||||
@@ -128,25 +133,33 @@ def test_last_layer_consumes_but_does_not_skip_the_pending_all_reduce(eligible):
|
||||
out_hidden, _ = last.prepare_attn(hidden_states, torch.zeros(8, 8), _DECODE)
|
||||
|
||||
assert torch.equal(out_hidden, torch.ones(8, 8))
|
||||
assert last._ffn_exit_fusions == (last._next_input_norm_takes_ffn_sum,)
|
||||
defer, absorb, _ = last._ffn_exit_fusions
|
||||
assert defer(_DECODE) is None and absorb(_DECODE) is None
|
||||
|
||||
|
||||
def test_cutedsl_entries_come_before_the_base_fused_kernel():
|
||||
comm = _communicator()
|
||||
assert comm._attn_input_fusions == (
|
||||
comm._finalize_output_and_update_and_read_residual_cutedsl,
|
||||
comm._reduce_output_and_update_and_read_residual_cutedsl,
|
||||
comm._reduce_output_and_update_and_read_residual,
|
||||
fusion = comm.fusions
|
||||
finalize, reduce, base_input = comm._attn_input_fusions
|
||||
assert _bound(finalize) == (
|
||||
fusion._finalize_output_and_update_and_read_residual,
|
||||
(comm,),
|
||||
)
|
||||
assert _bound(reduce) == (
|
||||
fusion._reduce_output_and_update_and_read_residual,
|
||||
(comm,),
|
||||
)
|
||||
assert base_input == comm._reduce_output_and_update_and_read_residual
|
||||
base = comm._mlp_input_reduce_output_and_update_and_read_residual
|
||||
cutedsl = comm._mlp_input_reduce_output_and_update_and_read_residual_cutedsl
|
||||
cutedsl = (fusion._mlp_input_reduce_output_and_update_and_read_residual, (comm,))
|
||||
with patch(
|
||||
f"{_MODULE}.get_parallel",
|
||||
return_value=SimpleNamespace(attn_tp_size=2, tp_size=2),
|
||||
):
|
||||
comm._declared = SimpleNamespace(input_rows=Layout(frozenset()))
|
||||
fusions = comm._select_mlp_input_fusions()
|
||||
assert [f.run for f in fusions] == [cutedsl, base]
|
||||
assert _bound(fusions[0].run) == cutedsl
|
||||
assert fusions[1].run == base
|
||||
# The workspace reduces over the TP group, which is the attention-TP group here.
|
||||
assert [f.completes for f in fusions] == [SumGroup.ATTN_TP] * 2
|
||||
assert all(f.may_return_new_residual for f in fusions)
|
||||
@@ -163,12 +176,12 @@ def test_the_fusion_runs_only_on_the_ffn_full_rows():
|
||||
the batch's FFN input on the full rows, not each rank's own slice (a2a, the
|
||||
fp4 all-gather, DWDP or a dense MLP on every rank)."""
|
||||
comm = _communicator()
|
||||
comm.fusion_service = SimpleNamespace(supports=lambda m: True)
|
||||
comm._context = SimpleNamespace(tp_size=2)
|
||||
fusion = comm.fusions
|
||||
fusion.service = SimpleNamespace(supports=lambda m: True)
|
||||
with (
|
||||
patch(
|
||||
f"{_MODULE}.get_parallel",
|
||||
return_value=SimpleNamespace(attn_cp_size=1),
|
||||
return_value=SimpleNamespace(attn_cp_size=1, tp_size=2),
|
||||
),
|
||||
patch(f"{_MODULE}.is_dp_attention_enabled", return_value=False),
|
||||
patch(
|
||||
@@ -187,22 +200,27 @@ def test_the_fusion_runs_only_on_the_ffn_full_rows():
|
||||
comm._batch_steps = lambda fb, rows=Layout(sharded): SimpleNamespace(
|
||||
ffn=SimpleNamespace(input_rows=rows)
|
||||
)
|
||||
assert comm._common_eligible(_DECODE, 8) is eligible
|
||||
assert fusion._common_eligible(comm, _DECODE, 8) is eligible
|
||||
|
||||
|
||||
def test_the_exit_tries_the_handoff_then_the_absorb_then_the_base_kernel():
|
||||
def test_the_exit_tries_the_handoff_then_the_absorb_then_the_base_kernel(eligible):
|
||||
comm = _communicator()
|
||||
comm.install(
|
||||
fusion = comm.fusions
|
||||
defer, absorb, base = comm._ffn_exit_fusions
|
||||
assert _bound(defer) == (fusion._defer_moe_finalize, (comm,))
|
||||
assert _bound(absorb) == (fusion._absorb_all_reduce, (comm,))
|
||||
assert base == comm._next_input_norm_takes_ffn_sum
|
||||
# Before install() chooses them, neither CuTe DSL kernel takes the sum.
|
||||
assert defer(_DECODE) is None and absorb(_DECODE) is None
|
||||
fusion.install(
|
||||
SimpleNamespace(),
|
||||
hands_off_finalize=True,
|
||||
next_input_absorbs=True,
|
||||
output_is_replicated=False,
|
||||
)
|
||||
assert comm._ffn_exit_fusions == (
|
||||
comm._defer_moe_finalize_cutedsl,
|
||||
comm._absorb_all_reduce_cutedsl,
|
||||
comm._next_input_norm_takes_ffn_sum,
|
||||
)
|
||||
with patch.object(CuteDSLFusion, "_should_use_finalize", return_value=True):
|
||||
assert defer(_DECODE) is FfnExitFusion.DEFER_MOE_FINALIZE
|
||||
assert absorb(_DECODE) is FfnExitFusion.NEXT_INPUT
|
||||
|
||||
|
||||
def test_a_replicated_output_producer_keeps_its_own_all_reduce(eligible):
|
||||
@@ -229,7 +247,10 @@ def test_a_replicated_output_producer_keeps_its_own_all_reduce(eligible):
|
||||
assert exit_fusion(replicated) is None
|
||||
assert exit_fusion(plain) is FfnExitFusion.NEXT_INPUT
|
||||
# Consuming what a predecessor skipped stays independently eligible.
|
||||
assert replicated._can_consume_post_moe_all_reduce(_DECODE, 8) is True
|
||||
assert (
|
||||
replicated.fusions._can_consume_post_moe_all_reduce(replicated, _DECODE, 8)
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
def test_a_service_nested_under_a_wrapper_is_prepared():
|
||||
@@ -241,7 +262,7 @@ def test_a_service_nested_under_a_wrapper_is_prepared():
|
||||
wrapper = torch.nn.Module()
|
||||
wrapper.language_model = torch.nn.Sequential(layer)
|
||||
_install([layer])
|
||||
layer.layer_communicator.fusion_service.prepare = lambda *, max_m: prepared.append(
|
||||
layer.layer_communicator.fusions.service.prepare = lambda *, max_m: prepared.append(
|
||||
max_m
|
||||
)
|
||||
|
||||
@@ -372,19 +393,15 @@ def test_a_handoff_the_kernel_does_not_take_is_completed_then_normed():
|
||||
|
||||
comm = _communicator()
|
||||
comm.input_layernorm = AddNorm()
|
||||
comm.fusion_service = SimpleNamespace(
|
||||
comm.fusions.service = SimpleNamespace(
|
||||
finalize=lambda **kw: pytest.fail("the kernel does not take this batch")
|
||||
)
|
||||
finish = MagicMock(return_value=torch.ones(2, 8))
|
||||
with (
|
||||
# A norm the kernel could fold in, on a batch it does not take.
|
||||
patch(f"{_MODULE}._fused_norm_gamma", return_value=torch.ones(8)),
|
||||
patch.object(
|
||||
CuteDSLFusionLayerCommunicator, "_should_use_finalize", return_value=False
|
||||
),
|
||||
patch.object(
|
||||
CuteDSLFusionLayerCommunicator, "_common_eligible", return_value=False
|
||||
),
|
||||
patch.object(CuteDSLFusion, "_should_use_finalize", return_value=False),
|
||||
patch.object(CuteDSLFusion, "_common_eligible", return_value=False),
|
||||
):
|
||||
hidden, residual = comm.prepare_attn(
|
||||
_handoff(finish), torch.full((2, 8), 2.0), _DECODE
|
||||
|
||||
Reference in New Issue
Block a user