From 06d012eacad7f0a79c8bf4467159af4fc6fcbc44 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Sun, 27 Sep 2026 16:55:47 -0700 Subject: [PATCH] [Refactor] Give a layer its CuTe DSL kernels at construction instead of a subclass (#41442) --- .../sglang/srt/layers/communicator/layer.py | 42 +++- .../srt/layers/moe/cutedsl_ar_fusion.py | 215 +++++++++--------- python/sglang/srt/models/deepseek_v2.py | 12 +- python/sglang/srt/models/qwen3_5.py | 17 +- .../unit/layers/moe/test_cutedsl_ar_fusion.py | 83 ++++--- 5 files changed, 208 insertions(+), 161 deletions(-) diff --git a/python/sglang/srt/layers/communicator/layer.py b/python/sglang/srt/layers/communicator/layer.py index 2fc59409c0..0f5ce9e8c7 100644 --- a/python/sglang/srt/layers/communicator/layer.py +++ b/python/sglang/srt/layers/communicator/layer.py @@ -16,7 +16,7 @@ from dataclasses import dataclass from enum import Enum, auto from functools import cached_property, partial -from typing import Callable, Optional, Tuple, Union +from typing import Callable, Optional, Protocol, Tuple, Union import msgspec import torch @@ -301,6 +301,22 @@ class StageCommunicator: return hidden_states, residual +class LayerFusions(Protocol): + """The fused kernels a fusion backend gives a layer, bound to it and tried + before the layer's own: at its attention input (each takes what the previous + layer left, the residual, the batch and a post-residual addition), at its + FFN input, and at its FFN exit (each says what the exit does when its kernel + takes the batch).""" + + def attention_input(self, layer: "LayerCommunicator") -> Tuple[Callable, ...]: ... + + def ffn_input(self, layer: "LayerCommunicator") -> Tuple["FusedMlpInput", ...]: ... + + def ffn_exit( + self, layer: "LayerCommunicator" + ) -> Tuple[Callable[[ForwardBatch], Optional["FfnExitFusion"]], ...]: ... + + class LayerCommunicator: # Communicators built without __init__ (e.g. test doubles) publish no LoRA # layout and try no fused kernel at the FFN exit. @@ -308,6 +324,8 @@ class LayerCommunicator: _ffn_exit_fusions: Tuple[Callable, ...] = () # A plain residual unless the layer is built with its own. _residual: LayerResidual = PLAIN_RESIDUAL + # The fused kernels a backend gives the layer, if any. + fusions: Optional[LayerFusions] = None def __init__( self, @@ -328,6 +346,8 @@ class LayerCommunicator: # A layer that is one stage of a sequence of stages, instead of an # attention followed by an FFN. stage: Optional["LayerStage"] = None, + # The fused kernels a backend gives the layer, tried before its own. + fusions: Optional[LayerFusions] = None, ): self.layer_scatter_modes = layer_scatter_modes self.input_layernorm = input_layernorm @@ -340,6 +360,7 @@ class LayerCommunicator: self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16 self.allow_deferred_ffn_reduction = allow_deferred_ffn_reduction self._residual = residual + self.fusions = fusions self._context = CommunicateContext.init_new() self._context.force_layernorm_before_dp_gather = ( @@ -765,9 +786,10 @@ class LayerCommunicator: Each takes (owed, residual, forward_batch, post_residual_addition) and returns None when it does not take the batch. They add the residual plainly; the boundary tries them only when the update it writes in is a - plain add.""" + plain add. A backend's come first.""" + given = self.fusions.attention_input(self) if self.fusions else () if not hasattr(self.input_layernorm, "forward_with_allreduce_fusion"): - return () + return given self._attn_input_fuses_quant = ( self.enable_fused_ar_quant and _use_aiter @@ -775,7 +797,7 @@ class LayerCommunicator: self.input_layernorm, "forward_with_allreduce_fusion_quant_per_group" ) ) - return (self._reduce_output_and_update_and_read_residual,) + return (*given, self._reduce_output_and_update_and_read_residual) def _reduce_output_and_update_and_read_residual( self, @@ -822,10 +844,13 @@ class LayerCommunicator: def _select_mlp_input_fusions(self) -> Tuple["FusedMlpInput", ...]: """The fused kernels that can take the attention -> FFN steps, in the order they are tried. They add the residual plainly; the boundary tries - them only when the update it writes in is a plain add.""" + them only when the update it writes in is a plain add. A backend's come + first.""" + given = self.fusions.ffn_input(self) if self.fusions else () if not hasattr(self.post_attention_layernorm, "forward_with_allreduce_fusion"): - return () + return given return ( + *given, FusedMlpInput( completes=SumGroup.ATTN_TP, run=self._mlp_input_reduce_output_and_update_and_read_residual, @@ -1123,8 +1148,9 @@ class LayerCommunicator: ) -> Tuple[Callable[[ForwardBatch], Optional["FfnExitFusion"]], ...]: """The fused kernels of the next layer's input that may take this layer's FFN sum, in the order they are tried. Each returns what the exit does - when its kernel takes this batch, or None.""" - return (self._next_input_norm_takes_ffn_sum,) + when its kernel takes this batch, or None. A backend's come first.""" + given = self.fusions.ffn_exit(self) if self.fusions else () + return (*given, self._next_input_norm_takes_ffn_sum) def _next_input_norm_takes_ffn_sum( self, forward_batch: ForwardBatch diff --git a/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py b/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py index e614ea0d9f..c473f73d00 100644 --- a/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py +++ b/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py @@ -8,6 +8,7 @@ shared-expert add folded in when the runner hands back a MoeFinalizeHandoff. from __future__ import annotations import logging +from functools import partial from typing import Callable, Optional, Sequence import torch @@ -195,11 +196,20 @@ class CuteDSLFusionService: ) -class CuteDSLFusionLayerCommunicator(LayerCommunicator): - fusion_service: CuteDSLFusionService | None = None - # The FFN exit's CuteDSL kernels, which install() gives from what it knows - # of this layer and the one after it. - _cutedsl_exit_fusions: tuple = () +class CuteDSLFusion: + """One layer's CuTe DSL kernels, given to its LayerCommunicator at + construction. At the attention input: the MoE finalize + AR + add + norm of + a handoff the previous layer left, and the AR + add + norm of a sum it left. + At the FFN input: the attention output's AR + add + norm. At the FFN exit: + handing the MoE finalize, or the all-reduce, to the next layer's kernel. + install_cutedsl_fusion gives it the model's service and chooses the exit + from the layer after this one; until then no kernel takes a batch.""" + + def __init__(self) -> None: + self.service: CuteDSLFusionService | None = None + # The FFN exit's choices, from what install() knows of the next layer. + self.hands_off_finalize = False + self.next_input_absorbs = False def install( self, @@ -209,162 +219,163 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator): next_input_absorbs: bool, output_is_replicated: bool, ) -> None: - """Take the model's fusion service and choose the FFN exit's CuteDSL - kernels: the MoE may hand off its finalize when it can and something - after it takes the handoff (``hands_off_finalize``); the FFN may leave - its all-reduce to the next layer's AR + norm when that layer has one, - unless a replicated output follows the reduction (moving it would scale - that by tp).""" - self.fusion_service = service - self._cutedsl_exit_fusions = ( - (self._defer_moe_finalize_cutedsl,) if hands_off_finalize else () - ) + ( - (self._absorb_all_reduce_cutedsl,) - if next_input_absorbs and not output_is_replicated - else () - ) - self._ffn_exit_fusions = self._select_ffn_exit_fusions() + """Take the model's fusion service and choose the FFN exit's kernels: + the MoE may hand off its finalize when it can and something after it + takes the handoff (``hands_off_finalize``); the FFN may leave its + all-reduce to the next layer's AR + norm when that layer has one, unless + a replicated output follows the reduction (moving it would scale that + by tp).""" + self.service = service + self.hands_off_finalize = hands_off_finalize + self.next_input_absorbs = next_input_absorbs and not output_is_replicated - def _select_attn_input_fusions(self): + def attention_input(self, layer: LayerCommunicator) -> tuple: return ( - self._finalize_output_and_update_and_read_residual_cutedsl, - self._reduce_output_and_update_and_read_residual_cutedsl, - *super()._select_attn_input_fusions(), + partial(self._finalize_output_and_update_and_read_residual, layer), + partial(self._reduce_output_and_update_and_read_residual, layer), ) - def _finalize_output_and_update_and_read_residual_cutedsl( - self, owed, residual, forward_batch, post_residual_addition + def ffn_input(self, layer: LayerCommunicator) -> tuple: + parallel = get_parallel() + if ( + TokenAxis.ATTN_TP_SCATTER not in layer.input_rows.sharded + # The workspace sums over TP, which is then the attention-TP group. + and parallel.attn_tp_size == parallel.tp_size + and _fused_norm_gamma(layer.post_attention_layernorm) is not None + ): + return ( + FusedMlpInput( + completes=SumGroup.ATTN_TP, + run=partial( + self._mlp_input_reduce_output_and_update_and_read_residual, + layer, + ), + may_return_new_residual=True, + ), + ) + return () + + def ffn_exit(self, layer: LayerCommunicator) -> tuple: + return ( + partial(self._defer_moe_finalize, layer), + partial(self._absorb_all_reduce, layer), + ) + + def _finalize_output_and_update_and_read_residual( + self, layer, owed, residual, forward_batch, post_residual_addition ): """Finish a deferred MoE finalize with the all-reduce, residual add and input norm, when this layer's kernel takes the batch; otherwise the handoff is completed by its producer's own tail.""" if not isinstance(owed, MoeFinalizeHandoff): return None - gamma = _fused_norm_gamma(self.input_layernorm) - if gamma is None or not self._should_use_finalize(forward_batch, owed.m): + gamma = _fused_norm_gamma(layer.input_layernorm) + if gamma is None or not self._should_use_finalize(layer, forward_batch, owed.m): return None if post_residual_addition is not None: residual = residual + post_residual_addition - assert self.fusion_service is not None - return self.fusion_service.finalize( - handoff=owed, residual=residual, gamma=gamma - ) + assert self.service is not None + return self.service.finalize(handoff=owed, residual=residual, gamma=gamma) - def _reduce_output_and_update_and_read_residual_cutedsl( - self, owed, residual, forward_batch, post_residual_addition + def _reduce_output_and_update_and_read_residual( + self, layer, owed, residual, forward_batch, post_residual_addition ): """Complete the all-reduce the previous layer left with the residual add and input norm.""" if not isinstance(owed, UnreducedOutput) or not ( self._can_consume_post_moe_all_reduce( - forward_batch, int(owed.partial.shape[0]) + layer, forward_batch, int(owed.partial.shape[0]) ) ): return None if post_residual_addition is not None: residual = residual + post_residual_addition - assert self.fusion_service is not None - return self.fusion_service.all_reduce_residual_rms_norm( + assert self.service is not None + return self.service.all_reduce_residual_rms_norm( local_contribution=owed.partial, residual=residual, - gamma=_fused_norm_gamma(self.input_layernorm), + gamma=_fused_norm_gamma(layer.input_layernorm), ) - def _select_mlp_input_fusions(self): - fusions = super()._select_mlp_input_fusions() - parallel = get_parallel() - if ( - TokenAxis.ATTN_TP_SCATTER not in self.input_rows.sharded - # The workspace sums over TP, which is then the attention-TP group. - and parallel.attn_tp_size == parallel.tp_size - and _fused_norm_gamma(self.post_attention_layernorm) is not None - ): - return ( - FusedMlpInput( - completes=SumGroup.ATTN_TP, - run=self._mlp_input_reduce_output_and_update_and_read_residual_cutedsl, - may_return_new_residual=True, - ), - *fusions, - ) - return fusions - - def _mlp_input_reduce_output_and_update_and_read_residual_cutedsl( - self, hidden_states, residual, forward_batch + def _mlp_input_reduce_output_and_update_and_read_residual( + self, layer, hidden_states, residual, forward_batch ): """The attention output's all-reduce with the residual add and the post-attention norm.""" if not ( - self._common_eligible(forward_batch, int(hidden_states.shape[0])) + self._common_eligible(layer, forward_batch, int(hidden_states.shape[0])) and residual is not None and not get_exec().comm.enable_quant_communications ): return None - assert self.fusion_service is not None - return self.fusion_service.all_reduce_residual_rms_norm( + assert self.service is not None + return self.service.all_reduce_residual_rms_norm( local_contribution=hidden_states, residual=residual, - gamma=_fused_norm_gamma(self.post_attention_layernorm), + gamma=_fused_norm_gamma(layer.post_attention_layernorm), ) - def _should_use_finalize(self, forward_batch: ForwardBatch, m: int) -> bool: + def _should_use_finalize(self, layer, forward_batch: ForwardBatch, m: int) -> bool: return ( - self._common_eligible(forward_batch, m) and get_parallel().moe_ep_size == 1 + self._common_eligible(layer, forward_batch, m) + and get_parallel().moe_ep_size == 1 ) def _can_consume_post_moe_all_reduce( - self, forward_batch: ForwardBatch, m: int + self, layer, forward_batch: ForwardBatch, m: int ) -> bool: """Incoming, and independent of this layer's own successor.""" return ( - self._common_eligible(forward_batch, m) - and _fused_norm_gamma(self.input_layernorm) is not None + self._common_eligible(layer, forward_batch, m) + and _fused_norm_gamma(layer.input_layernorm) is not None and not get_exec().comm.enable_quant_communications ) - def _select_ffn_exit_fusions(self): - return (*self._cutedsl_exit_fusions, *super()._select_ffn_exit_fusions()) - - def _defer_moe_finalize_cutedsl( - self, forward_batch: ForwardBatch + def _defer_moe_finalize( + self, layer, forward_batch: ForwardBatch ) -> Optional[FfnExitFusion]: """The MoE hands its unfinalized output to the next layer's finalize + all-reduce + norm.""" - if self._should_use_finalize( - forward_batch, int(forward_batch.input_ids.shape[0]) + if self.hands_off_finalize and self._should_use_finalize( + layer, forward_batch, int(forward_batch.input_ids.shape[0]) ): return FfnExitFusion.DEFER_MOE_FINALIZE return None - def _absorb_all_reduce_cutedsl( - self, forward_batch: ForwardBatch + def _absorb_all_reduce( + self, layer, forward_batch: ForwardBatch ) -> Optional[FfnExitFusion]: """The next layer's AR + norm takes the post-experts all-reduce.""" - if self._can_consume_post_moe_all_reduce( - forward_batch, int(forward_batch.input_ids.shape[0]) + if self.next_input_absorbs and self._can_consume_post_moe_all_reduce( + layer, forward_batch, int(forward_batch.input_ids.shape[0]) ): return FfnExitFusion.NEXT_INPUT return None - def _common_eligible(self, forward_batch: ForwardBatch, m: int) -> bool: + def _common_eligible(self, layer, forward_batch: ForwardBatch, m: int) -> bool: parallel = get_parallel() return bool( - self.fusion_service is not None + self.service is not None and _is_supported_forward_mode(forward_batch.forward_mode) - and self.fusion_service.supports(m) + and self.service.supports(m) and not is_dp_attention_enabled() # Also forces moe_dp_size == 1, so a hybrid EP x MoE-TP reduction # always merges into the one TP reduction the workspace performs. and parallel.attn_cp_size == 1 and not get_attn_tp_context().input_scattered and get_moe_a2a_backend().is_none() - and self._context.tp_size > 1 + and parallel.tp_size > 1 # The FFN runs on the full rows, not each rank's own slice. and TokenAxis.ATTN_TP_SCATTER - not in self._batch_steps(forward_batch).ffn.input_rows.sharded + not in layer.ffn.entry(forward_batch).input_rows.sharded ) +def _fusion_of(layer: torch.nn.Module) -> CuteDSLFusion | None: + fusions = getattr(layer.layer_communicator, "fusions", None) + return fusions if isinstance(fusions, CuteDSLFusion) else None + + def install_cutedsl_fusion( layers: Sequence[torch.nn.Module], *, @@ -384,11 +395,7 @@ def install_cutedsl_fusion( # A draft shares the target's process, which holds one workspace. return None - fusion_layers = [ - layer - for layer in layers - if isinstance(layer.layer_communicator, CuteDSLFusionLayerCommunicator) - ] + fusion_layers = [layer for layer in layers if _fusion_of(layer) is not None] if not fusion_layers: return None @@ -414,26 +421,22 @@ def install_cutedsl_fusion( ) hands_off = 0 for index, layer in enumerate(layers): - communicator = layer.layer_communicator - if not isinstance(communicator, CuteDSLFusionLayerCommunicator): + fusion = _fusion_of(layer) + if fusion is None: continue successor = layers[index + 1] if index + 1 < len(layers) else None if successor is None: has_consumer = final_norm_consumes_handoff else: - has_consumer = isinstance( - successor.layer_communicator, CuteDSLFusionLayerCommunicator - ) + has_consumer = _fusion_of(successor) is not None hands_off_finalize = bool(can_defer_finalize(layer)) and has_consumer hands_off += hands_off_finalize - communicator.install( + fusion.install( service, hands_off_finalize=hands_off_finalize, # False on the last layer: the final norm does not all-reduce. next_input_absorbs=successor is not None - and isinstance( - successor.layer_communicator, CuteDSLFusionLayerCommunicator - ), + and _fusion_of(successor) is not None, output_is_replicated=requires_local_reduction is not None and bool(requires_local_reduction(layer)), ) @@ -456,16 +459,18 @@ def prepare_cutedsl_fusion( Scans the module tree, so a wrapper holding the model as a submodule (a VLM's language_model) needs no hook of its own. """ - communicators = [ - communicator + fusions = [ + fusion for module in model.modules() # Most modules carry no layer_communicator. if isinstance( - communicator := module.__dict__.get("layer_communicator"), - CuteDSLFusionLayerCommunicator, + fusion := getattr( + module.__dict__.get("layer_communicator"), "fusions", None + ), + CuteDSLFusion, ) ] - if not communicators or any(c.fusion_service is None for c in communicators): + if not fusions or any(f.service is None for f in fusions): raise ValueError( "--flashinfer-allreduce-fusion-backend cutedsl is set, but " f"{type(model).__name__} installed no CuTe DSL fusion service, so no " @@ -477,7 +482,7 @@ def prepare_cutedsl_fusion( "FlashInfer MNNVL CuTe DSL fusion does not support concurrent PDMux " "streams sharing one mutable workspace" ) - services = {id(c.fusion_service): c.fusion_service for c in communicators} + services = {id(f.service): f.service for f in fusions} max_m = _resolve_max_m(max_running_requests=max_running_requests) for service in services.values(): service.prepare(max_m=max_m) @@ -485,5 +490,5 @@ def prepare_cutedsl_fusion( "Prepared the FlashInfer MNNVL CuTe DSL fusion workspace for M_max=%d " "(%d fused layers)", max_m, - len(communicators), + len(fusions), ) diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index b19df05080..0408a9ec7e 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -2632,26 +2632,24 @@ class DeepseekV2DecoderLayer(nn.Module): ): """The communicator for this layer's norms; it chooses its boundary steps from them at construction.""" + fusions = None if ( not get_parallel().enable_prefill_cp and not self.is_nextn and _use_mnnvl_cutedsl_fusion() ): # Dense layers too: selecting cutedsl turns the legacy fusion off. - from sglang.srt.layers.moe.cutedsl_ar_fusion import ( - CuteDSLFusionLayerCommunicator, - ) + from sglang.srt.layers.moe.cutedsl_ar_fusion import CuteDSLFusion - communicator_cls = CuteDSLFusionLayerCommunicator - else: - communicator_cls = LayerCommunicator - return communicator_cls( + fusions = CuteDSLFusion() + return LayerCommunicator( layer_scatter_modes=self.layer_scatter_modes, input_layernorm=input_layernorm, post_attention_layernorm=post_attention_layernorm, allow_reduce_scatter=True, qkv_latent_func=qkv_latent_func, allow_deferred_ffn_reduction=allow_deferred_ffn_reduction, + fusions=fusions, ) def _detect_gfx95_quant_format(self) -> str: diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 30e2011fe2..92a6211753 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -214,14 +214,13 @@ def _use_mnnvl_cutedsl_fusion(config: Qwen3_5TextConfig, is_nextn: bool) -> bool ) -def _layer_communicator_class(config: Qwen3_5TextConfig, is_nextn: bool): +def _layer_fusions(config: Qwen3_5TextConfig, is_nextn: bool): + """The CuTe DSL kernels a layer tries before its own, when they are on.""" if _use_mnnvl_cutedsl_fusion(config, is_nextn): - from sglang.srt.layers.moe.cutedsl_ar_fusion import ( - CuteDSLFusionLayerCommunicator, - ) + from sglang.srt.layers.moe.cutedsl_ar_fusion import CuteDSLFusion - return CuteDSLFusionLayerCommunicator - return LayerCommunicator + return CuteDSLFusion() + return None if _is_cuda: @@ -1124,13 +1123,14 @@ class Qwen3_5LinearDecoderLayer(nn.Module): _enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz) ) - self.layer_communicator = _layer_communicator_class(config, is_nextn)( + self.layer_communicator = LayerCommunicator( layer_scatter_modes=self.layer_scatter_modes, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, enable_fused_ar_quant=enable_fused_ar_quant, fused_ar_quant_keep_bf16=enable_fused_ar_quant, + fusions=_layer_fusions(config, is_nextn), ) def forward( @@ -1327,13 +1327,14 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): enable_fused_ar_quant = ( _enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.qkv_proj) ) - self.layer_communicator = _layer_communicator_class(config, is_nextn)( + self.layer_communicator = LayerCommunicator( layer_scatter_modes=self.layer_scatter_modes, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, enable_fused_ar_quant=enable_fused_ar_quant, fused_ar_quant_keep_bf16=False, + fusions=_layer_fusions(config, is_nextn), ) self.alt_stream = alt_stream diff --git a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py index bdbf70c1e7..5a62ace744 100644 --- a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py +++ b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py @@ -24,7 +24,7 @@ from sglang.srt.layers.flashinfer_mnnvl_cutedsl import ( ) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.moe.cutedsl_ar_fusion import ( - CuteDSLFusionLayerCommunicator, + CuteDSLFusion, MoeFinalizeHandoff, install_cutedsl_fusion, prepare_cutedsl_fusion, @@ -43,11 +43,13 @@ _DECODE = SimpleNamespace(forward_mode=ForwardMode.DECODE, input_ids=torch.zeros def _communicator(): - comm = CuteDSLFusionLayerCommunicator.__new__(CuteDSLFusionLayerCommunicator) + comm = LayerCommunicator.__new__(LayerCommunicator) + comm.fusions = CuteDSLFusion() comm.input_layernorm = RMSNorm(8, eps=1e-6) comm.post_attention_layernorm = RMSNorm(8, eps=1e-6) comm.enable_fused_ar_quant = False comm._attn_input_fusions = comm._select_attn_input_fusions() + comm._ffn_exit_fusions = comm._select_ffn_exit_fusions() # Only the ordinary batches' attention input half, with these entries. comm._context = SimpleNamespace() comm._sp_steps = comm._input_scattered_steps = comm._cp_steps = None @@ -71,6 +73,11 @@ def _communicator(): return comm +def _bound(entry): + """What an entry runs and the layer it is bound to.""" + return entry.func, entry.args + + def _install(layers, **kwargs): reset_context() publish(ServerArgs(model_path="dummy"), role="test") @@ -89,9 +96,7 @@ def _install(layers, **kwargs): def eligible(): """Every shared gate open, so a case varies only the predicate it names.""" with ( - patch.object( - CuteDSLFusionLayerCommunicator, "_common_eligible", return_value=True - ), + patch.object(CuteDSLFusion, "_common_eligible", return_value=True), patch( f"{_MODULE}.get_exec", return_value=SimpleNamespace( @@ -106,7 +111,7 @@ def test_last_layer_consumes_but_does_not_skip_the_pending_all_reduce(eligible): """The last layer must fuse the reduction its predecessor skipped, or the unreduced output reaches the final norm; it must not skip its own.""" last = _communicator() - last.install( + last.fusions.install( SimpleNamespace( all_reduce_residual_rms_norm=lambda *, local_contribution, residual, gamma: ( local_contribution + 1, @@ -128,25 +133,33 @@ def test_last_layer_consumes_but_does_not_skip_the_pending_all_reduce(eligible): out_hidden, _ = last.prepare_attn(hidden_states, torch.zeros(8, 8), _DECODE) assert torch.equal(out_hidden, torch.ones(8, 8)) - assert last._ffn_exit_fusions == (last._next_input_norm_takes_ffn_sum,) + defer, absorb, _ = last._ffn_exit_fusions + assert defer(_DECODE) is None and absorb(_DECODE) is None def test_cutedsl_entries_come_before_the_base_fused_kernel(): comm = _communicator() - assert comm._attn_input_fusions == ( - comm._finalize_output_and_update_and_read_residual_cutedsl, - comm._reduce_output_and_update_and_read_residual_cutedsl, - comm._reduce_output_and_update_and_read_residual, + fusion = comm.fusions + finalize, reduce, base_input = comm._attn_input_fusions + assert _bound(finalize) == ( + fusion._finalize_output_and_update_and_read_residual, + (comm,), ) + assert _bound(reduce) == ( + fusion._reduce_output_and_update_and_read_residual, + (comm,), + ) + assert base_input == comm._reduce_output_and_update_and_read_residual base = comm._mlp_input_reduce_output_and_update_and_read_residual - cutedsl = comm._mlp_input_reduce_output_and_update_and_read_residual_cutedsl + cutedsl = (fusion._mlp_input_reduce_output_and_update_and_read_residual, (comm,)) with patch( f"{_MODULE}.get_parallel", return_value=SimpleNamespace(attn_tp_size=2, tp_size=2), ): comm._declared = SimpleNamespace(input_rows=Layout(frozenset())) fusions = comm._select_mlp_input_fusions() - assert [f.run for f in fusions] == [cutedsl, base] + assert _bound(fusions[0].run) == cutedsl + assert fusions[1].run == base # The workspace reduces over the TP group, which is the attention-TP group here. assert [f.completes for f in fusions] == [SumGroup.ATTN_TP] * 2 assert all(f.may_return_new_residual for f in fusions) @@ -163,12 +176,12 @@ def test_the_fusion_runs_only_on_the_ffn_full_rows(): the batch's FFN input on the full rows, not each rank's own slice (a2a, the fp4 all-gather, DWDP or a dense MLP on every rank).""" comm = _communicator() - comm.fusion_service = SimpleNamespace(supports=lambda m: True) - comm._context = SimpleNamespace(tp_size=2) + fusion = comm.fusions + fusion.service = SimpleNamespace(supports=lambda m: True) with ( patch( f"{_MODULE}.get_parallel", - return_value=SimpleNamespace(attn_cp_size=1), + return_value=SimpleNamespace(attn_cp_size=1, tp_size=2), ), patch(f"{_MODULE}.is_dp_attention_enabled", return_value=False), patch( @@ -187,22 +200,27 @@ def test_the_fusion_runs_only_on_the_ffn_full_rows(): comm._batch_steps = lambda fb, rows=Layout(sharded): SimpleNamespace( ffn=SimpleNamespace(input_rows=rows) ) - assert comm._common_eligible(_DECODE, 8) is eligible + assert fusion._common_eligible(comm, _DECODE, 8) is eligible -def test_the_exit_tries_the_handoff_then_the_absorb_then_the_base_kernel(): +def test_the_exit_tries_the_handoff_then_the_absorb_then_the_base_kernel(eligible): comm = _communicator() - comm.install( + fusion = comm.fusions + defer, absorb, base = comm._ffn_exit_fusions + assert _bound(defer) == (fusion._defer_moe_finalize, (comm,)) + assert _bound(absorb) == (fusion._absorb_all_reduce, (comm,)) + assert base == comm._next_input_norm_takes_ffn_sum + # Before install() chooses them, neither CuTe DSL kernel takes the sum. + assert defer(_DECODE) is None and absorb(_DECODE) is None + fusion.install( SimpleNamespace(), hands_off_finalize=True, next_input_absorbs=True, output_is_replicated=False, ) - assert comm._ffn_exit_fusions == ( - comm._defer_moe_finalize_cutedsl, - comm._absorb_all_reduce_cutedsl, - comm._next_input_norm_takes_ffn_sum, - ) + with patch.object(CuteDSLFusion, "_should_use_finalize", return_value=True): + assert defer(_DECODE) is FfnExitFusion.DEFER_MOE_FINALIZE + assert absorb(_DECODE) is FfnExitFusion.NEXT_INPUT def test_a_replicated_output_producer_keeps_its_own_all_reduce(eligible): @@ -229,7 +247,10 @@ def test_a_replicated_output_producer_keeps_its_own_all_reduce(eligible): assert exit_fusion(replicated) is None assert exit_fusion(plain) is FfnExitFusion.NEXT_INPUT # Consuming what a predecessor skipped stays independently eligible. - assert replicated._can_consume_post_moe_all_reduce(_DECODE, 8) is True + assert ( + replicated.fusions._can_consume_post_moe_all_reduce(replicated, _DECODE, 8) + is True + ) def test_a_service_nested_under_a_wrapper_is_prepared(): @@ -241,7 +262,7 @@ def test_a_service_nested_under_a_wrapper_is_prepared(): wrapper = torch.nn.Module() wrapper.language_model = torch.nn.Sequential(layer) _install([layer]) - layer.layer_communicator.fusion_service.prepare = lambda *, max_m: prepared.append( + layer.layer_communicator.fusions.service.prepare = lambda *, max_m: prepared.append( max_m ) @@ -372,19 +393,15 @@ def test_a_handoff_the_kernel_does_not_take_is_completed_then_normed(): comm = _communicator() comm.input_layernorm = AddNorm() - comm.fusion_service = SimpleNamespace( + comm.fusions.service = SimpleNamespace( finalize=lambda **kw: pytest.fail("the kernel does not take this batch") ) finish = MagicMock(return_value=torch.ones(2, 8)) with ( # A norm the kernel could fold in, on a batch it does not take. patch(f"{_MODULE}._fused_norm_gamma", return_value=torch.ones(8)), - patch.object( - CuteDSLFusionLayerCommunicator, "_should_use_finalize", return_value=False - ), - patch.object( - CuteDSLFusionLayerCommunicator, "_common_eligible", return_value=False - ), + patch.object(CuteDSLFusion, "_should_use_finalize", return_value=False), + patch.object(CuteDSLFusion, "_common_eligible", return_value=False), ): hidden, residual = comm.prepare_attn( _handoff(finish), torch.full((2, 8), 2.0), _DECODE