[Refactor] Give a layer its CuTe DSL kernels at construction instead of a subclass (#41442)
This commit is contained in:
@@ -16,7 +16,7 @@
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from functools import cached_property, partial
|
||||
from typing import Callable, Optional, Tuple, Union
|
||||
from typing import Callable, Optional, Protocol, Tuple, Union
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
@@ -301,6 +301,22 @@ class StageCommunicator:
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
class LayerFusions(Protocol):
|
||||
"""The fused kernels a fusion backend gives a layer, bound to it and tried
|
||||
before the layer's own: at its attention input (each takes what the previous
|
||||
layer left, the residual, the batch and a post-residual addition), at its
|
||||
FFN input, and at its FFN exit (each says what the exit does when its kernel
|
||||
takes the batch)."""
|
||||
|
||||
def attention_input(self, layer: "LayerCommunicator") -> Tuple[Callable, ...]: ...
|
||||
|
||||
def ffn_input(self, layer: "LayerCommunicator") -> Tuple["FusedMlpInput", ...]: ...
|
||||
|
||||
def ffn_exit(
|
||||
self, layer: "LayerCommunicator"
|
||||
) -> Tuple[Callable[[ForwardBatch], Optional["FfnExitFusion"]], ...]: ...
|
||||
|
||||
|
||||
class LayerCommunicator:
|
||||
# Communicators built without __init__ (e.g. test doubles) publish no LoRA
|
||||
# layout and try no fused kernel at the FFN exit.
|
||||
@@ -308,6 +324,8 @@ class LayerCommunicator:
|
||||
_ffn_exit_fusions: Tuple[Callable, ...] = ()
|
||||
# A plain residual unless the layer is built with its own.
|
||||
_residual: LayerResidual = PLAIN_RESIDUAL
|
||||
# The fused kernels a backend gives the layer, if any.
|
||||
fusions: Optional[LayerFusions] = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -328,6 +346,8 @@ class LayerCommunicator:
|
||||
# A layer that is one stage of a sequence of stages, instead of an
|
||||
# attention followed by an FFN.
|
||||
stage: Optional["LayerStage"] = None,
|
||||
# The fused kernels a backend gives the layer, tried before its own.
|
||||
fusions: Optional[LayerFusions] = None,
|
||||
):
|
||||
self.layer_scatter_modes = layer_scatter_modes
|
||||
self.input_layernorm = input_layernorm
|
||||
@@ -340,6 +360,7 @@ class LayerCommunicator:
|
||||
self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16
|
||||
self.allow_deferred_ffn_reduction = allow_deferred_ffn_reduction
|
||||
self._residual = residual
|
||||
self.fusions = fusions
|
||||
|
||||
self._context = CommunicateContext.init_new()
|
||||
self._context.force_layernorm_before_dp_gather = (
|
||||
@@ -765,9 +786,10 @@ class LayerCommunicator:
|
||||
Each takes (owed, residual, forward_batch, post_residual_addition) and
|
||||
returns None when it does not take the batch. They add the residual
|
||||
plainly; the boundary tries them only when the update it writes in is a
|
||||
plain add."""
|
||||
plain add. A backend's come first."""
|
||||
given = self.fusions.attention_input(self) if self.fusions else ()
|
||||
if not hasattr(self.input_layernorm, "forward_with_allreduce_fusion"):
|
||||
return ()
|
||||
return given
|
||||
self._attn_input_fuses_quant = (
|
||||
self.enable_fused_ar_quant
|
||||
and _use_aiter
|
||||
@@ -775,7 +797,7 @@ class LayerCommunicator:
|
||||
self.input_layernorm, "forward_with_allreduce_fusion_quant_per_group"
|
||||
)
|
||||
)
|
||||
return (self._reduce_output_and_update_and_read_residual,)
|
||||
return (*given, self._reduce_output_and_update_and_read_residual)
|
||||
|
||||
def _reduce_output_and_update_and_read_residual(
|
||||
self,
|
||||
@@ -822,10 +844,13 @@ class LayerCommunicator:
|
||||
def _select_mlp_input_fusions(self) -> Tuple["FusedMlpInput", ...]:
|
||||
"""The fused kernels that can take the attention -> FFN steps, in the
|
||||
order they are tried. They add the residual plainly; the boundary tries
|
||||
them only when the update it writes in is a plain add."""
|
||||
them only when the update it writes in is a plain add. A backend's come
|
||||
first."""
|
||||
given = self.fusions.ffn_input(self) if self.fusions else ()
|
||||
if not hasattr(self.post_attention_layernorm, "forward_with_allreduce_fusion"):
|
||||
return ()
|
||||
return given
|
||||
return (
|
||||
*given,
|
||||
FusedMlpInput(
|
||||
completes=SumGroup.ATTN_TP,
|
||||
run=self._mlp_input_reduce_output_and_update_and_read_residual,
|
||||
@@ -1123,8 +1148,9 @@ class LayerCommunicator:
|
||||
) -> Tuple[Callable[[ForwardBatch], Optional["FfnExitFusion"]], ...]:
|
||||
"""The fused kernels of the next layer's input that may take this layer's
|
||||
FFN sum, in the order they are tried. Each returns what the exit does
|
||||
when its kernel takes this batch, or None."""
|
||||
return (self._next_input_norm_takes_ffn_sum,)
|
||||
when its kernel takes this batch, or None. A backend's come first."""
|
||||
given = self.fusions.ffn_exit(self) if self.fusions else ()
|
||||
return (*given, self._next_input_norm_takes_ffn_sum)
|
||||
|
||||
def _next_input_norm_takes_ffn_sum(
|
||||
self, forward_batch: ForwardBatch
|
||||
|
||||
@@ -8,6 +8,7 @@ shared-expert add folded in when the runner hands back a MoeFinalizeHandoff.
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from functools import partial
|
||||
from typing import Callable, Optional, Sequence
|
||||
|
||||
import torch
|
||||
@@ -195,11 +196,20 @@ class CuteDSLFusionService:
|
||||
)
|
||||
|
||||
|
||||
class CuteDSLFusionLayerCommunicator(LayerCommunicator):
|
||||
fusion_service: CuteDSLFusionService | None = None
|
||||
# The FFN exit's CuteDSL kernels, which install() gives from what it knows
|
||||
# of this layer and the one after it.
|
||||
_cutedsl_exit_fusions: tuple = ()
|
||||
class CuteDSLFusion:
|
||||
"""One layer's CuTe DSL kernels, given to its LayerCommunicator at
|
||||
construction. At the attention input: the MoE finalize + AR + add + norm of
|
||||
a handoff the previous layer left, and the AR + add + norm of a sum it left.
|
||||
At the FFN input: the attention output's AR + add + norm. At the FFN exit:
|
||||
handing the MoE finalize, or the all-reduce, to the next layer's kernel.
|
||||
install_cutedsl_fusion gives it the model's service and chooses the exit
|
||||
from the layer after this one; until then no kernel takes a batch."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.service: CuteDSLFusionService | None = None
|
||||
# The FFN exit's choices, from what install() knows of the next layer.
|
||||
self.hands_off_finalize = False
|
||||
self.next_input_absorbs = False
|
||||
|
||||
def install(
|
||||
self,
|
||||
@@ -209,162 +219,163 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator):
|
||||
next_input_absorbs: bool,
|
||||
output_is_replicated: bool,
|
||||
) -> None:
|
||||
"""Take the model's fusion service and choose the FFN exit's CuteDSL
|
||||
kernels: the MoE may hand off its finalize when it can and something
|
||||
after it takes the handoff (``hands_off_finalize``); the FFN may leave
|
||||
its all-reduce to the next layer's AR + norm when that layer has one,
|
||||
unless a replicated output follows the reduction (moving it would scale
|
||||
that by tp)."""
|
||||
self.fusion_service = service
|
||||
self._cutedsl_exit_fusions = (
|
||||
(self._defer_moe_finalize_cutedsl,) if hands_off_finalize else ()
|
||||
) + (
|
||||
(self._absorb_all_reduce_cutedsl,)
|
||||
if next_input_absorbs and not output_is_replicated
|
||||
else ()
|
||||
)
|
||||
self._ffn_exit_fusions = self._select_ffn_exit_fusions()
|
||||
"""Take the model's fusion service and choose the FFN exit's kernels:
|
||||
the MoE may hand off its finalize when it can and something after it
|
||||
takes the handoff (``hands_off_finalize``); the FFN may leave its
|
||||
all-reduce to the next layer's AR + norm when that layer has one, unless
|
||||
a replicated output follows the reduction (moving it would scale that
|
||||
by tp)."""
|
||||
self.service = service
|
||||
self.hands_off_finalize = hands_off_finalize
|
||||
self.next_input_absorbs = next_input_absorbs and not output_is_replicated
|
||||
|
||||
def _select_attn_input_fusions(self):
|
||||
def attention_input(self, layer: LayerCommunicator) -> tuple:
|
||||
return (
|
||||
self._finalize_output_and_update_and_read_residual_cutedsl,
|
||||
self._reduce_output_and_update_and_read_residual_cutedsl,
|
||||
*super()._select_attn_input_fusions(),
|
||||
partial(self._finalize_output_and_update_and_read_residual, layer),
|
||||
partial(self._reduce_output_and_update_and_read_residual, layer),
|
||||
)
|
||||
|
||||
def _finalize_output_and_update_and_read_residual_cutedsl(
|
||||
self, owed, residual, forward_batch, post_residual_addition
|
||||
def ffn_input(self, layer: LayerCommunicator) -> tuple:
|
||||
parallel = get_parallel()
|
||||
if (
|
||||
TokenAxis.ATTN_TP_SCATTER not in layer.input_rows.sharded
|
||||
# The workspace sums over TP, which is then the attention-TP group.
|
||||
and parallel.attn_tp_size == parallel.tp_size
|
||||
and _fused_norm_gamma(layer.post_attention_layernorm) is not None
|
||||
):
|
||||
return (
|
||||
FusedMlpInput(
|
||||
completes=SumGroup.ATTN_TP,
|
||||
run=partial(
|
||||
self._mlp_input_reduce_output_and_update_and_read_residual,
|
||||
layer,
|
||||
),
|
||||
may_return_new_residual=True,
|
||||
),
|
||||
)
|
||||
return ()
|
||||
|
||||
def ffn_exit(self, layer: LayerCommunicator) -> tuple:
|
||||
return (
|
||||
partial(self._defer_moe_finalize, layer),
|
||||
partial(self._absorb_all_reduce, layer),
|
||||
)
|
||||
|
||||
def _finalize_output_and_update_and_read_residual(
|
||||
self, layer, owed, residual, forward_batch, post_residual_addition
|
||||
):
|
||||
"""Finish a deferred MoE finalize with the all-reduce, residual add and
|
||||
input norm, when this layer's kernel takes the batch; otherwise the
|
||||
handoff is completed by its producer's own tail."""
|
||||
if not isinstance(owed, MoeFinalizeHandoff):
|
||||
return None
|
||||
gamma = _fused_norm_gamma(self.input_layernorm)
|
||||
if gamma is None or not self._should_use_finalize(forward_batch, owed.m):
|
||||
gamma = _fused_norm_gamma(layer.input_layernorm)
|
||||
if gamma is None or not self._should_use_finalize(layer, forward_batch, owed.m):
|
||||
return None
|
||||
if post_residual_addition is not None:
|
||||
residual = residual + post_residual_addition
|
||||
assert self.fusion_service is not None
|
||||
return self.fusion_service.finalize(
|
||||
handoff=owed, residual=residual, gamma=gamma
|
||||
)
|
||||
assert self.service is not None
|
||||
return self.service.finalize(handoff=owed, residual=residual, gamma=gamma)
|
||||
|
||||
def _reduce_output_and_update_and_read_residual_cutedsl(
|
||||
self, owed, residual, forward_batch, post_residual_addition
|
||||
def _reduce_output_and_update_and_read_residual(
|
||||
self, layer, owed, residual, forward_batch, post_residual_addition
|
||||
):
|
||||
"""Complete the all-reduce the previous layer left with the residual add
|
||||
and input norm."""
|
||||
if not isinstance(owed, UnreducedOutput) or not (
|
||||
self._can_consume_post_moe_all_reduce(
|
||||
forward_batch, int(owed.partial.shape[0])
|
||||
layer, forward_batch, int(owed.partial.shape[0])
|
||||
)
|
||||
):
|
||||
return None
|
||||
if post_residual_addition is not None:
|
||||
residual = residual + post_residual_addition
|
||||
assert self.fusion_service is not None
|
||||
return self.fusion_service.all_reduce_residual_rms_norm(
|
||||
assert self.service is not None
|
||||
return self.service.all_reduce_residual_rms_norm(
|
||||
local_contribution=owed.partial,
|
||||
residual=residual,
|
||||
gamma=_fused_norm_gamma(self.input_layernorm),
|
||||
gamma=_fused_norm_gamma(layer.input_layernorm),
|
||||
)
|
||||
|
||||
def _select_mlp_input_fusions(self):
|
||||
fusions = super()._select_mlp_input_fusions()
|
||||
parallel = get_parallel()
|
||||
if (
|
||||
TokenAxis.ATTN_TP_SCATTER not in self.input_rows.sharded
|
||||
# The workspace sums over TP, which is then the attention-TP group.
|
||||
and parallel.attn_tp_size == parallel.tp_size
|
||||
and _fused_norm_gamma(self.post_attention_layernorm) is not None
|
||||
):
|
||||
return (
|
||||
FusedMlpInput(
|
||||
completes=SumGroup.ATTN_TP,
|
||||
run=self._mlp_input_reduce_output_and_update_and_read_residual_cutedsl,
|
||||
may_return_new_residual=True,
|
||||
),
|
||||
*fusions,
|
||||
)
|
||||
return fusions
|
||||
|
||||
def _mlp_input_reduce_output_and_update_and_read_residual_cutedsl(
|
||||
self, hidden_states, residual, forward_batch
|
||||
def _mlp_input_reduce_output_and_update_and_read_residual(
|
||||
self, layer, hidden_states, residual, forward_batch
|
||||
):
|
||||
"""The attention output's all-reduce with the residual add and the
|
||||
post-attention norm."""
|
||||
if not (
|
||||
self._common_eligible(forward_batch, int(hidden_states.shape[0]))
|
||||
self._common_eligible(layer, forward_batch, int(hidden_states.shape[0]))
|
||||
and residual is not None
|
||||
and not get_exec().comm.enable_quant_communications
|
||||
):
|
||||
return None
|
||||
assert self.fusion_service is not None
|
||||
return self.fusion_service.all_reduce_residual_rms_norm(
|
||||
assert self.service is not None
|
||||
return self.service.all_reduce_residual_rms_norm(
|
||||
local_contribution=hidden_states,
|
||||
residual=residual,
|
||||
gamma=_fused_norm_gamma(self.post_attention_layernorm),
|
||||
gamma=_fused_norm_gamma(layer.post_attention_layernorm),
|
||||
)
|
||||
|
||||
def _should_use_finalize(self, forward_batch: ForwardBatch, m: int) -> bool:
|
||||
def _should_use_finalize(self, layer, forward_batch: ForwardBatch, m: int) -> bool:
|
||||
return (
|
||||
self._common_eligible(forward_batch, m) and get_parallel().moe_ep_size == 1
|
||||
self._common_eligible(layer, forward_batch, m)
|
||||
and get_parallel().moe_ep_size == 1
|
||||
)
|
||||
|
||||
def _can_consume_post_moe_all_reduce(
|
||||
self, forward_batch: ForwardBatch, m: int
|
||||
self, layer, forward_batch: ForwardBatch, m: int
|
||||
) -> bool:
|
||||
"""Incoming, and independent of this layer's own successor."""
|
||||
return (
|
||||
self._common_eligible(forward_batch, m)
|
||||
and _fused_norm_gamma(self.input_layernorm) is not None
|
||||
self._common_eligible(layer, forward_batch, m)
|
||||
and _fused_norm_gamma(layer.input_layernorm) is not None
|
||||
and not get_exec().comm.enable_quant_communications
|
||||
)
|
||||
|
||||
def _select_ffn_exit_fusions(self):
|
||||
return (*self._cutedsl_exit_fusions, *super()._select_ffn_exit_fusions())
|
||||
|
||||
def _defer_moe_finalize_cutedsl(
|
||||
self, forward_batch: ForwardBatch
|
||||
def _defer_moe_finalize(
|
||||
self, layer, forward_batch: ForwardBatch
|
||||
) -> Optional[FfnExitFusion]:
|
||||
"""The MoE hands its unfinalized output to the next layer's finalize +
|
||||
all-reduce + norm."""
|
||||
if self._should_use_finalize(
|
||||
forward_batch, int(forward_batch.input_ids.shape[0])
|
||||
if self.hands_off_finalize and self._should_use_finalize(
|
||||
layer, forward_batch, int(forward_batch.input_ids.shape[0])
|
||||
):
|
||||
return FfnExitFusion.DEFER_MOE_FINALIZE
|
||||
return None
|
||||
|
||||
def _absorb_all_reduce_cutedsl(
|
||||
self, forward_batch: ForwardBatch
|
||||
def _absorb_all_reduce(
|
||||
self, layer, forward_batch: ForwardBatch
|
||||
) -> Optional[FfnExitFusion]:
|
||||
"""The next layer's AR + norm takes the post-experts all-reduce."""
|
||||
if self._can_consume_post_moe_all_reduce(
|
||||
forward_batch, int(forward_batch.input_ids.shape[0])
|
||||
if self.next_input_absorbs and self._can_consume_post_moe_all_reduce(
|
||||
layer, forward_batch, int(forward_batch.input_ids.shape[0])
|
||||
):
|
||||
return FfnExitFusion.NEXT_INPUT
|
||||
return None
|
||||
|
||||
def _common_eligible(self, forward_batch: ForwardBatch, m: int) -> bool:
|
||||
def _common_eligible(self, layer, forward_batch: ForwardBatch, m: int) -> bool:
|
||||
parallel = get_parallel()
|
||||
return bool(
|
||||
self.fusion_service is not None
|
||||
self.service is not None
|
||||
and _is_supported_forward_mode(forward_batch.forward_mode)
|
||||
and self.fusion_service.supports(m)
|
||||
and self.service.supports(m)
|
||||
and not is_dp_attention_enabled()
|
||||
# Also forces moe_dp_size == 1, so a hybrid EP x MoE-TP reduction
|
||||
# always merges into the one TP reduction the workspace performs.
|
||||
and parallel.attn_cp_size == 1
|
||||
and not get_attn_tp_context().input_scattered
|
||||
and get_moe_a2a_backend().is_none()
|
||||
and self._context.tp_size > 1
|
||||
and parallel.tp_size > 1
|
||||
# The FFN runs on the full rows, not each rank's own slice.
|
||||
and TokenAxis.ATTN_TP_SCATTER
|
||||
not in self._batch_steps(forward_batch).ffn.input_rows.sharded
|
||||
not in layer.ffn.entry(forward_batch).input_rows.sharded
|
||||
)
|
||||
|
||||
|
||||
def _fusion_of(layer: torch.nn.Module) -> CuteDSLFusion | None:
|
||||
fusions = getattr(layer.layer_communicator, "fusions", None)
|
||||
return fusions if isinstance(fusions, CuteDSLFusion) else None
|
||||
|
||||
|
||||
def install_cutedsl_fusion(
|
||||
layers: Sequence[torch.nn.Module],
|
||||
*,
|
||||
@@ -384,11 +395,7 @@ def install_cutedsl_fusion(
|
||||
# A draft shares the target's process, which holds one workspace.
|
||||
return None
|
||||
|
||||
fusion_layers = [
|
||||
layer
|
||||
for layer in layers
|
||||
if isinstance(layer.layer_communicator, CuteDSLFusionLayerCommunicator)
|
||||
]
|
||||
fusion_layers = [layer for layer in layers if _fusion_of(layer) is not None]
|
||||
if not fusion_layers:
|
||||
return None
|
||||
|
||||
@@ -414,26 +421,22 @@ def install_cutedsl_fusion(
|
||||
)
|
||||
hands_off = 0
|
||||
for index, layer in enumerate(layers):
|
||||
communicator = layer.layer_communicator
|
||||
if not isinstance(communicator, CuteDSLFusionLayerCommunicator):
|
||||
fusion = _fusion_of(layer)
|
||||
if fusion is None:
|
||||
continue
|
||||
successor = layers[index + 1] if index + 1 < len(layers) else None
|
||||
if successor is None:
|
||||
has_consumer = final_norm_consumes_handoff
|
||||
else:
|
||||
has_consumer = isinstance(
|
||||
successor.layer_communicator, CuteDSLFusionLayerCommunicator
|
||||
)
|
||||
has_consumer = _fusion_of(successor) is not None
|
||||
hands_off_finalize = bool(can_defer_finalize(layer)) and has_consumer
|
||||
hands_off += hands_off_finalize
|
||||
communicator.install(
|
||||
fusion.install(
|
||||
service,
|
||||
hands_off_finalize=hands_off_finalize,
|
||||
# False on the last layer: the final norm does not all-reduce.
|
||||
next_input_absorbs=successor is not None
|
||||
and isinstance(
|
||||
successor.layer_communicator, CuteDSLFusionLayerCommunicator
|
||||
),
|
||||
and _fusion_of(successor) is not None,
|
||||
output_is_replicated=requires_local_reduction is not None
|
||||
and bool(requires_local_reduction(layer)),
|
||||
)
|
||||
@@ -456,16 +459,18 @@ def prepare_cutedsl_fusion(
|
||||
Scans the module tree, so a wrapper holding the model as a submodule (a VLM's
|
||||
language_model) needs no hook of its own.
|
||||
"""
|
||||
communicators = [
|
||||
communicator
|
||||
fusions = [
|
||||
fusion
|
||||
for module in model.modules()
|
||||
# Most modules carry no layer_communicator.
|
||||
if isinstance(
|
||||
communicator := module.__dict__.get("layer_communicator"),
|
||||
CuteDSLFusionLayerCommunicator,
|
||||
fusion := getattr(
|
||||
module.__dict__.get("layer_communicator"), "fusions", None
|
||||
),
|
||||
CuteDSLFusion,
|
||||
)
|
||||
]
|
||||
if not communicators or any(c.fusion_service is None for c in communicators):
|
||||
if not fusions or any(f.service is None for f in fusions):
|
||||
raise ValueError(
|
||||
"--flashinfer-allreduce-fusion-backend cutedsl is set, but "
|
||||
f"{type(model).__name__} installed no CuTe DSL fusion service, so no "
|
||||
@@ -477,7 +482,7 @@ def prepare_cutedsl_fusion(
|
||||
"FlashInfer MNNVL CuTe DSL fusion does not support concurrent PDMux "
|
||||
"streams sharing one mutable workspace"
|
||||
)
|
||||
services = {id(c.fusion_service): c.fusion_service for c in communicators}
|
||||
services = {id(f.service): f.service for f in fusions}
|
||||
max_m = _resolve_max_m(max_running_requests=max_running_requests)
|
||||
for service in services.values():
|
||||
service.prepare(max_m=max_m)
|
||||
@@ -485,5 +490,5 @@ def prepare_cutedsl_fusion(
|
||||
"Prepared the FlashInfer MNNVL CuTe DSL fusion workspace for M_max=%d "
|
||||
"(%d fused layers)",
|
||||
max_m,
|
||||
len(communicators),
|
||||
len(fusions),
|
||||
)
|
||||
|
||||
@@ -2632,26 +2632,24 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
):
|
||||
"""The communicator for this layer's norms; it chooses its boundary
|
||||
steps from them at construction."""
|
||||
fusions = None
|
||||
if (
|
||||
not get_parallel().enable_prefill_cp
|
||||
and not self.is_nextn
|
||||
and _use_mnnvl_cutedsl_fusion()
|
||||
):
|
||||
# Dense layers too: selecting cutedsl turns the legacy fusion off.
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import (
|
||||
CuteDSLFusionLayerCommunicator,
|
||||
)
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import CuteDSLFusion
|
||||
|
||||
communicator_cls = CuteDSLFusionLayerCommunicator
|
||||
else:
|
||||
communicator_cls = LayerCommunicator
|
||||
return communicator_cls(
|
||||
fusions = CuteDSLFusion()
|
||||
return LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
input_layernorm=input_layernorm,
|
||||
post_attention_layernorm=post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
qkv_latent_func=qkv_latent_func,
|
||||
allow_deferred_ffn_reduction=allow_deferred_ffn_reduction,
|
||||
fusions=fusions,
|
||||
)
|
||||
|
||||
def _detect_gfx95_quant_format(self) -> str:
|
||||
|
||||
@@ -214,14 +214,13 @@ def _use_mnnvl_cutedsl_fusion(config: Qwen3_5TextConfig, is_nextn: bool) -> bool
|
||||
)
|
||||
|
||||
|
||||
def _layer_communicator_class(config: Qwen3_5TextConfig, is_nextn: bool):
|
||||
def _layer_fusions(config: Qwen3_5TextConfig, is_nextn: bool):
|
||||
"""The CuTe DSL kernels a layer tries before its own, when they are on."""
|
||||
if _use_mnnvl_cutedsl_fusion(config, is_nextn):
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import (
|
||||
CuteDSLFusionLayerCommunicator,
|
||||
)
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import CuteDSLFusion
|
||||
|
||||
return CuteDSLFusionLayerCommunicator
|
||||
return LayerCommunicator
|
||||
return CuteDSLFusion()
|
||||
return None
|
||||
|
||||
|
||||
if _is_cuda:
|
||||
@@ -1124,13 +1123,14 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
_enable_qwen35_fused_ar_quant()
|
||||
and _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
|
||||
)
|
||||
self.layer_communicator = _layer_communicator_class(config, is_nextn)(
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
enable_fused_ar_quant=enable_fused_ar_quant,
|
||||
fused_ar_quant_keep_bf16=enable_fused_ar_quant,
|
||||
fusions=_layer_fusions(config, is_nextn),
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -1327,13 +1327,14 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
enable_fused_ar_quant = (
|
||||
_enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.qkv_proj)
|
||||
)
|
||||
self.layer_communicator = _layer_communicator_class(config, is_nextn)(
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
enable_fused_ar_quant=enable_fused_ar_quant,
|
||||
fused_ar_quant_keep_bf16=False,
|
||||
fusions=_layer_fusions(config, is_nextn),
|
||||
)
|
||||
|
||||
self.alt_stream = alt_stream
|
||||
|
||||
Reference in New Issue
Block a user