diff --git a/python/sglang/srt/arg_groups/fields/exec_.py b/python/sglang/srt/arg_groups/fields/exec_.py index 7da4dca844..95705ead56 100644 --- a/python/sglang/srt/arg_groups/fields/exec_.py +++ b/python/sglang/srt/arg_groups/fields/exec_.py @@ -630,7 +630,7 @@ class ExecComm(msgspec.Struct): "Enforce disable FlashInfer allreduce fusion.", ] = False flashinfer_allreduce_fusion_backend: A[ - Optional[Literal["auto", "trtllm", "mnnvl"]], + Optional[Literal["auto", "trtllm", "mnnvl", "cutedsl"]], Arg( help=( "Enable FlashInfer allreduce fusion and choose backend. " @@ -641,6 +641,9 @@ class ExecComm(msgspec.Struct): "'trtllm': available on single-node systems only. " "'mnnvl': available on SM90 single-node systems and SM100/SM103 " "single-node or multi-node systems via MNNVL fabric. " + "'cutedsl': Blackwell-only bf16 MNNVL CuTe DSL backend; also " + "fuses the MoE finalize and the shared-expert add into the " + "collective when the MoE runner can defer them. " "Fuses allreduce with Residual + RMSNorm for supported MoE models." ), resolvable=True, diff --git a/python/sglang/srt/arg_groups/moe_hook.py b/python/sglang/srt/arg_groups/moe_hook.py index d8e9ae02df..599cef5f10 100644 --- a/python/sglang/srt/arg_groups/moe_hook.py +++ b/python/sglang/srt/arg_groups/moe_hook.py @@ -202,6 +202,7 @@ def validate_flashinfer_megamoe_model(server_args: Any) -> None: "DeepseekV32ForCausalLM", "DeepseekV4ForCausalLM", "Glm4MoeForCausalLM", + "GlmMoeDsaForCausalLM", "NemotronHForCausalLM", "NemotronHPuzzleForCausalLM", "Qwen2MoeForCausalLM", diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py index 53418a5237..efc05c91ae 100644 --- a/python/sglang/srt/arg_groups/overrides.py +++ b/python/sglang/srt/arg_groups/overrides.py @@ -920,20 +920,6 @@ def _flashinfer_allreduce_fusion_auto_enable(view: Any) -> dict: and view.nnodes == 1 and not view.disable_custom_all_reduce ) - if envs.SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION.get() and model_arch in { - "Qwen3_5MoeForCausalLM", - "Qwen3_5MoeForConditionalGeneration", - }: - # The Qwen backend owns one workspace for ordinary AR and MoE finalize; - # do not allocate or fall back to the legacy TRTLLM/MNNVL workspace. - if view.flashinfer_allreduce_fusion_backend is not None: - logger.warning( - "SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION owns both Qwen3.5 " - "AllReduce fusion patterns; suppressing the separately configured " - "--flashinfer-allreduce-fusion-backend=%s", - view.flashinfer_allreduce_fusion_backend, - ) - return {"flashinfer_allreduce_fusion_backend": None} if ( view.flashinfer_allreduce_fusion_backend is None and model_arch in _FLASHINFER_ALLREDUCE_FUSION_ARCHS diff --git a/python/sglang/srt/distributed/bootstrap.py b/python/sglang/srt/distributed/bootstrap.py index 6630962729..dce7f01009 100644 --- a/python/sglang/srt/distributed/bootstrap.py +++ b/python/sglang/srt/distributed/bootstrap.py @@ -236,8 +236,12 @@ def _set_all_reduce_flags() -> None: set_custom_all_reduce(not get_exec().comm.disable_custom_all_reduce) set_mscclpp_all_reduce(get_exec().comm.enable_mscclpp) set_torch_symm_mem_all_reduce(get_exec().comm.enable_torch_symm_mem) + from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion + set_flashinfer_allreduce_only( get_exec().comm.flashinfer_allreduce_fusion_backend is not None + # cutedsl has no legacy workspace for tagged groups to reduce over. + and not uses_cutedsl_ar_fusion() ) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index adc7d08ed2..0c700596ae 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1244,6 +1244,9 @@ class Envs: SGLANG_NIXL_EP_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(128) SGLANG_PPLX_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(128) SGLANG_ENABLE_MOE_DEFERRED_FINALIZE = EnvBool(True) + # The [M*top_k, hidden] HBM round trip pays only at small M; 0 disables. + # GLM-5.2 (H=6144): neutral at 192 on B300 TP8, crossover 192-223 on GB300 TP4. + SGLANG_MOE_DEFERRED_FINALIZE_MAX_TOKENS = EnvInt(192) # DeepSeek/GLM MoE (deepseek_v2.py): quantize the (dp-gathered) MoE input # to per-token-group-128 fp8 ONCE and feed both the fused shared-expert # GEMM (cutlass w8a8 linear) and the routed experts' triton fused runner, @@ -1769,13 +1772,6 @@ class Envs: # Qwen3.5 and GDN SGLANG_ENABLE_GDN_DECODE_FUSED_PROJ_CONV = EnvBool(True) - SGLANG_TRACE_QWEN35_FINAL_NORM = EnvBool(False) - SGLANG_QWEN35_NATIVE_FINAL_NORM = EnvBool(False) - # One switch enables deferred MoE finalize and AR + residual + RMSNorm. - SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION = EnvBool(False) - # Distinct workspace configurations allowed in one process. Production - # uses one model/configuration per rank, so fail closed on accidental reuse. - SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION_MAX_INSTANCES = EnvInt(1) # =================================================================== # Plugin system @@ -1887,10 +1883,22 @@ class _DeprecatedEnv: # ad-hoc warnings. For a rename where the old name must keep working through a # descriptor, use EnvBoolWithAlias / EnvIntWithAlias instead. _DEPRECATED_ENVS: Dict[str, _DeprecatedEnv] = { + "SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION": _DeprecatedEnv( + note=( + "Pass --flashinfer-allreduce-fusion-backend cutedsl instead. " + "Without it an eligible model auto-enables the legacy mnnvl " + "backend rather than the CuTe DSL fusion." + ) + ), + "SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION_MAX_INSTANCES": _DeprecatedEnv( + note="One workspace per process is now an invariant, not a limit." + ), # Removed without replacement. "SGLANG_ENABLE_CP_V2": _DeprecatedEnv( note="Strategy-based prefill context parallelism is now the only generic implementation." ), + "SGLANG_TRACE_QWEN35_FINAL_NORM": _DeprecatedEnv(), + "SGLANG_QWEN35_NATIVE_FINAL_NORM": _DeprecatedEnv(), "SGLANG_ENABLE_HICACHE_BUFFER_ANCHOR_LOCK": _DeprecatedEnv( note="Buffer-mode anchor pinning is always on; set " "SGLANG_HICACHE_BUFFER_ANCHOR_LOCK_CAP=0 to disable it." diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index 2e02e6bb72..d4c0430c2a 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -58,7 +58,10 @@ from sglang.srt.layers.dp_attention import ( is_enable_moe_cp_allgather, moe_cp_all_gather_into_tensor, ) -from sglang.srt.layers.flashinfer_comm_fusion import is_flashinfer_allreduce_unavailable +from sglang.srt.layers.flashinfer_comm_fusion import ( + is_flashinfer_allreduce_unavailable, + uses_cutedsl_ar_fusion, +) from sglang.srt.layers.moe import ( can_merge_post_experts_all_reduce, deferred_post_experts_all_reduce, @@ -198,6 +201,8 @@ def apply_flashinfer_allreduce_fusion(batch_size: int): and _is_flashinfer_available and not is_dp_attention_enabled() and get_exec().comm.flashinfer_allreduce_fusion_backend is not None + # cutedsl runs its own fused path from the fusion communicator. + and not uses_cutedsl_ar_fusion() and not is_flashinfer_allreduce_unavailable() # Symbolic size checks stay last: under Dynamo tracing they guard on # the dynamic token dim, so statically-off configs must short-circuit @@ -859,6 +864,14 @@ class LayerCommunicator: post_residual_addition, ) + return self._finish_prepare_attn( + hidden_states=hidden_states, + residual=residual, + forward_batch=forward_batch, + ) + + def _finish_prepare_attn(self, hidden_states, residual, forward_batch): + """Tail every prepare_attn path must run, or ``attn_inputs`` is unset.""" hidden_states = self._communicate_simple_fn( hidden_states=hidden_states, forward_batch=forward_batch, @@ -953,6 +966,12 @@ class LayerCommunicator: return True return False + def should_defer_moe_finalize( + self, forward_batch: ForwardBatch, m: int | None = None + ) -> bool: + """Whether the MoE may hand an unfinalized output to the next layer.""" + return False + # NOTE: This function will cause torch recompilation def should_fuse_mlp_allreduce_with_next_layer( self, forward_batch: ForwardBatch @@ -1046,11 +1065,13 @@ def scatter_mode_layouts( class FfnExit: """One FFN's reduction decision. Inside the ``with`` block it is published as - ``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` on ``get_forward()``.""" + ``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` / ``defer_moe_finalize`` on + ``get_forward()``.""" __slots__ = ( "communicator", "forward_batch", + "defer_moe_finalize", "fuse_mlp_allreduce", "mlp_reduce_scatter", "_scope", @@ -1059,13 +1080,17 @@ class FfnExit: def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch): self.communicator = communicator self.forward_batch = forward_batch + self.defer_moe_finalize = communicator.should_defer_moe_finalize(forward_batch) + # Deferring implies fusing: a handoff skips the post-experts all-reduce. self.fuse_mlp_allreduce = ( - communicator.should_fuse_mlp_allreduce_with_next_layer(forward_batch) + self.defer_moe_finalize + or communicator.should_fuse_mlp_allreduce_with_next_layer(forward_batch) ) self.mlp_reduce_scatter = communicator.should_use_reduce_scatter(forward_batch) self._scope = get_forward().scoped( fuse_mlp_allreduce=self.fuse_mlp_allreduce, mlp_reduce_scatter=self.mlp_reduce_scatter, + defer_moe_finalize=self.defer_moe_finalize, ) def __enter__(self) -> "FfnExit": @@ -1079,6 +1104,10 @@ class FfnExit: self, hidden_states: torch.Tensor, residual: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: """Leave the reduction to the next layer's input norm, or postprocess.""" + if not isinstance(hidden_states, torch.Tensor): + # A deferred MoE finalize handoff, consumed by the next prepare_attn. + assert self.defer_moe_finalize, "unrequested deferred MoE handoff" + return hidden_states, residual if self.fuse_mlp_allreduce: hidden_states._sglang_needs_allreduce_fusion = True return hidden_states, residual diff --git a/python/sglang/srt/layers/flashinfer_comm_fusion.py b/python/sglang/srt/layers/flashinfer_comm_fusion.py index cdea99e216..d7b59408b2 100644 --- a/python/sglang/srt/layers/flashinfer_comm_fusion.py +++ b/python/sglang/srt/layers/flashinfer_comm_fusion.py @@ -47,6 +47,14 @@ def _resolve_backend(backend: str, is_multi_node: bool = False) -> str: "FlashInfer allreduce fusion requires SM90 or SM10X NVIDIA GPUs." ) + if backend == "cutedsl": + if not get_platform().is_sm100: + raise ValueError( + "FlashInfer allreduce fusion cutedsl backend requires a " + "Blackwell system." + ) + return backend + if backend == "auto": if is_multi_node: if get_platform().is_sm100: @@ -72,6 +80,11 @@ def _resolve_backend(backend: str, is_multi_node: bool = False) -> str: return backend +def uses_cutedsl_ar_fusion() -> bool: + """Selected CuTe DSL owns both patterns, so the legacy workspace stands down.""" + return get_exec().comm.flashinfer_allreduce_fusion_backend == "cutedsl" + + def resolve_flashinfer_allreduce_fusion_backend() -> Optional[str]: """The fusion backend for this process, or None when fusion is off. @@ -713,7 +726,7 @@ def ensure_workspace_initialized( use_attn_tp_group: bool = True, ): """Ensure workspace is initialized.""" - if _flashinfer_allreduce_unavailable: + if _flashinfer_allreduce_unavailable or uses_cutedsl_ar_fusion(): return False if not is_flashinfer_available() or _flashinfer_comm is None: diff --git a/python/sglang/srt/layers/flashinfer_mnnvl_cutedsl.py b/python/sglang/srt/layers/flashinfer_mnnvl_cutedsl.py index c824e4c0c8..2d630c1d61 100644 --- a/python/sglang/srt/layers/flashinfer_mnnvl_cutedsl.py +++ b/python/sglang/srt/layers/flashinfer_mnnvl_cutedsl.py @@ -3,9 +3,7 @@ from __future__ import annotations import logging -import threading -from dataclasses import dataclass, replace -from functools import lru_cache +from dataclasses import replace from typing import TYPE_CHECKING import torch @@ -20,8 +18,7 @@ logger = logging.getLogger(__name__) def _import_kernel_backend(): - # Imported here rather than at module scope: the CuTe DSL backend drags in - # CUDA-only dependencies that CPU-side importers of this module never need. + # Deferred: the backend drags in CUDA-only deps CPU importers never need. from flashinfer.comm import AllReduceFusionPattern, allreduce_fusion from flashinfer.comm.mnnvl_cutedsl import DEFAULT_CONFIG from flashinfer.comm.mnnvl_cutedsl_ar import MNNVLCuteDSLAllReduceFusionWorkspace @@ -34,6 +31,252 @@ def _import_kernel_backend(): ) +# Mirrors the FlashInfer HT device kernel: warp size, bf16 elements per 16B vector. +_WARP_SIZE = 32 +_VEC_BF16 = 8 + +# Kernel bound, with two non-consumer warps beside the reduction warps: +# consumer_threads + (2 + reduction_warps) * WARP_SIZE <= _CUDA_BLOCK_THREADS. +_CUDA_BLOCK_THREADS = 1024 +_HT_NON_REDUCTION_WARPS = 2 +_HT_REDUCTION_WARP_CHOICES = (1, 2, 4, 8) + +_SUPPORTED_TP_SIZES = (2, 4, 8, 16) + + +def _ht_shard_split( + hidden_size: int, max_consumer_threads: int +) -> tuple[int, int] | None: + # The kernel shards a token into consumer_threads * 8 * vectors_per_thread + # elements, and consumer_threads must divide its 16-byte vector count. + packs = hidden_size // _VEC_BF16 + # packs // 2 keeps vectors_per_thread >= 2, as in the GB300 presets. + limit = min(max_consumer_threads, packs // 2) + for consumer_threads in range( + limit - limit % _WARP_SIZE, _WARP_SIZE - 1, -_WARP_SIZE + ): + if packs % consumer_threads == 0: + return consumer_threads, packs // consumer_threads + return None + + +def _ht_reduction_warp_order(preferred: int) -> tuple[int, ...]: + # Preset's value first, then nearest, fewer before more: each warp costs consumers. + return tuple( + sorted( + _HT_REDUCTION_WARP_CHOICES, + key=lambda warps: (warps > preferred, abs(warps - preferred)), + ) + ) + + +def _ht_shard_major_is_legal( + consumer_threads: int, rms_token_groups: int, tp_size: int +) -> bool: + # Shard-major RMS needs an integer number of reduction shards per RMS warp. + rms_warps_per_token = (consumer_threads // rms_token_groups) // _WARP_SIZE + return ( + rms_warps_per_token > 0 + and tp_size >= rms_warps_per_token + and tp_size % rms_warps_per_token == 0 + ) + + +def _ht_retarget(preset, *, hidden_size: int, tp_size: int): + """``preset`` re-aimed at this shape, or None when no legal split exists. + + Only shape-dependent fields move, except ``rms_shard_major``, which the + kernel rejects unless tp is a multiple of the RMS warps per token. + """ + packs = hidden_size // _VEC_BF16 + if packs % tp_size: + # Kernel: "hidden vector count must be divisible by tp". + return None + packs_per_shard = packs // tp_size + for reduction_warps in _ht_reduction_warp_order(preset.reduction_warps): + if packs_per_shard % (reduction_warps * _WARP_SIZE): + continue + split = _ht_shard_split( + hidden_size, + _CUDA_BLOCK_THREADS + - (_HT_NON_REDUCTION_WARPS + reduction_warps) * _WARP_SIZE, + ) + if split is None: + continue + consumer_threads, vectors_per_thread = split + return replace( + preset, + consumer_threads=consumer_threads, + vectors_per_thread=vectors_per_thread, + reduction_warps=reduction_warps, + rms_shard_major=preset.rms_shard_major + and _ht_shard_major_is_legal( + consumer_threads, preset.rms_token_groups, tp_size + ), + ) + return None + + +def _routes(bounds, ll_target, bt_targets, ht_target): + # Without HT the widest BT range takes the unbounded slot, so the profile + # still covers the whole workspace capacity. + from flashinfer.comm.mnnvl_cutedsl import MRangeDispatch + + if ht_target is not None: + return MRangeDispatch( + upper_bounds=bounds, + targets=(ll_target, *bt_targets, ht_target), + ) + return MRangeDispatch( + upper_bounds=(*bounds[: len(bt_targets)], None), + targets=(ll_target, *bt_targets), + ) + + +def _gb300_presets(*, wide_tp: bool): + """The shipped GB300 (bounds, LL, BT pair, HT) per operation, TP16 or TP8.""" + from flashinfer.comm.mnnvl_cutedsl.kernel_bt import ( + BT_ALL_REDUCE_GB300_TP8_H8192_PRESET_0, + BT_ALL_REDUCE_GB300_TP8_H8192_PRESET_1, + BT_ALL_REDUCE_GB300_TP16_H8192_PRESET_0, + BT_ALL_REDUCE_GB300_TP16_H8192_PRESET_1, + BT_FINALIZE_GB300_TP8_H8192_K10_PRESET_0, + BT_FINALIZE_GB300_TP8_H8192_K10_PRESET_1, + BT_FINALIZE_GB300_TP16_H8192_K10_PRESET_0, + BT_FINALIZE_GB300_TP16_H8192_K10_PRESET_1, + ) + from flashinfer.comm.mnnvl_cutedsl.kernel_ht import ( + HT_ALL_REDUCE_GB300_TP8_H8192, + HT_ALL_REDUCE_GB300_TP16_H8192, + HT_FINALIZE_GB300_TP8_H8192_K10, + HT_FINALIZE_GB300_TP16_H8192_K10, + ) + from flashinfer.comm.mnnvl_cutedsl.kernel_ll import ( + LL_ALL_REDUCE_GB300_TP8_H8192, + LL_ALL_REDUCE_GB300_TP16_H8192, + LL_FINALIZE_GB300_TP8_H8192_K10, + LL_FINALIZE_GB300_TP16_H8192_K10, + ) + + # FlashInfer's measured GB300 crossovers. + if wide_tp: + return { + "finalize": ( + (7, 52, 703, None), + LL_FINALIZE_GB300_TP16_H8192_K10, + ( + BT_FINALIZE_GB300_TP16_H8192_K10_PRESET_0, + BT_FINALIZE_GB300_TP16_H8192_K10_PRESET_1, + ), + HT_FINALIZE_GB300_TP16_H8192_K10, + ), + "all_reduce": ( + (5, 512, 959, None), + LL_ALL_REDUCE_GB300_TP16_H8192, + ( + BT_ALL_REDUCE_GB300_TP16_H8192_PRESET_0, + BT_ALL_REDUCE_GB300_TP16_H8192_PRESET_1, + ), + HT_ALL_REDUCE_GB300_TP16_H8192, + ), + } + return { + "finalize": ( + (23, 48, 703, None), + LL_FINALIZE_GB300_TP8_H8192_K10, + ( + BT_FINALIZE_GB300_TP8_H8192_K10_PRESET_0, + BT_FINALIZE_GB300_TP8_H8192_K10_PRESET_1, + ), + HT_FINALIZE_GB300_TP8_H8192_K10, + ), + "all_reduce": ( + (15, 256, 1024, None), + LL_ALL_REDUCE_GB300_TP8_H8192, + ( + BT_ALL_REDUCE_GB300_TP8_H8192_PRESET_0, + BT_ALL_REDUCE_GB300_TP8_H8192_PRESET_1, + ), + HT_ALL_REDUCE_GB300_TP8_H8192, + ), + } + + +def _retargeted_config(tp_size: int, hidden_size: int, top_k: int): + """A single-profile routing config for a shape FlashInfer does not ship. + + Reuses the shipped GB300 crossovers, which were measured at tp=8/16 + hidden=8192 top_k=10 and are unmeasured elsewhere. + """ + from flashinfer.comm.mnnvl_cutedsl import ( + KernelTarget, + MNNVLCuteDSLConfig, + ProtocolKind, + StaticProfile, + ) + + shipped = _gb300_presets(wide_tp=tp_size >= 16) + ht = { + op: _ht_retarget(presets[3], hidden_size=hidden_size, tp_size=tp_size) + for op, presets in shipped.items() + } + if any(preset is None for preset in ht.values()): + # Both operations share the HT protocol state, so HT goes for both. + ht = dict.fromkeys(ht) + logger.warning( + "MNNVL CuTe DSL: hidden_size=%d admits no HT shard split at " + "tp_size=%d; serving this shape with the LL and BT routes only, " + "which costs throughput at large token counts.", + hidden_size, + tp_size, + ) + + def routes(op): + bounds, ll, bt, _ = shipped[op] + return _routes( + bounds, + KernelTarget(protocol=ProtocolKind.LL, preset=ll), + tuple(KernelTarget(protocol=ProtocolKind.BT, preset=p) for p in bt), + None + if ht[op] is None + else KernelTarget(protocol=ProtocolKind.HT, preset=ht[op]), + ) + + profile = StaticProfile( + tp_size=tp_size, + hidden_size=hidden_size, + top_k=top_k, + dtype=torch.bfloat16, + finalize_routes=routes("finalize"), + all_reduce_routes=routes("all_reduce"), + ) + return MNNVLCuteDSLConfig(profiles=(profile,)) + + +def _config_for_shape(default_config, *, tp_size: int, hidden_size: int, top_k: int): + # MNNVLCuteDSLConfig.resolve matches (tp, hidden, top_k, dtype) exactly and + # DEFAULT_CONFIG ships GB300 H=8192/K=10 only, so other shapes need one. + for profile in default_config.profiles: + if profile.matches( + tp_size=tp_size, + hidden_size=hidden_size, + top_k=top_k, + dtype=torch.bfloat16, + ): + return default_config + logger.info( + "MNNVL CuTe DSL: no shipped profile for tp=%d hidden=%d top_k=%d; " + "re-targeting the GB300 presets at this shape. Their M crossovers were " + "measured at tp=8/16 hidden=8192 top_k=10, so benchmark against " + "--flashinfer-allreduce-fusion-backend mnnvl before deploying a shape " + "that matters.", + tp_size, + hidden_size, + top_k, + ) + return _retargeted_config(tp_size, hidden_size, top_k) + + def _with_early_finalize_shared_load(config): profiles = [] updated_presets = 0 @@ -64,17 +307,6 @@ def _with_early_finalize_shared_load(config): return replace(config, profiles=tuple(profiles)) -@dataclass(frozen=True, slots=True) -class _WorkspaceSignature: - hidden_size: int - top_k: int - rms_epsilon: float - weight_bias: float - max_m: int - device_index: int - process_group_identity: int - - class FlashInferMNNVLCuteDSLARFusion: """One graph-stable workspace serving both supported fusion patterns.""" @@ -100,8 +332,14 @@ class FlashInferMNNVLCuteDSLARFusion: self.rms_epsilon = float(rms_epsilon) self.weight_bias = float(weight_bias) self.process_group = process_group + # Cached: supports() runs per layer and must not re-enter c10d. + self.tp_size = dist.get_world_size(process_group) + if self.tp_size not in _SUPPORTED_TP_SIZES: + raise ValueError( + f"MNNVL CuTe DSL fusion supports tp_size in {_SUPPORTED_TP_SIZES}, " + f"got {self.tp_size}" + ) self.device = torch.device(device) - self._destroyed = False with torch.cuda.device(self.device): self.device = torch.device("cuda", torch.cuda.current_device()) @@ -129,11 +367,14 @@ class FlashInferMNNVLCuteDSLARFusion: self._patterns, default_config, ) = _import_kernel_backend() - # Only fused finalize launches have a completed shared-expert handoff; - # standalone AllReduce kernels retain the safe load ordering. - + shaped_config = _config_for_shape( + default_config, + tp_size=self.tp_size, + hidden_size=self.hidden_size, + top_k=self.top_k, + ) if get_spec().speculative_algorithm is None: - self.workspace_config = _with_early_finalize_shared_load(default_config) + self.workspace_config = _with_early_finalize_shared_load(shaped_config) else: # Early shared load is safe only for a single looping decode graph; # alternating draft/verify replays can read an unfinished buffer. @@ -142,9 +383,9 @@ class FlashInferMNNVLCuteDSLARFusion: "CuTe DSL finalize presets on the safe (non-early-load) " "ordering." ) - self.workspace_config = default_config + self.workspace_config = shaped_config self.workspace = workspace_type( - tp_size=dist.get_world_size(process_group), + tp_size=self.tp_size, tp_rank=dist.get_rank(process_group), max_token_num=self.max_m, hidden_dim=self.hidden_size, @@ -166,10 +407,10 @@ class FlashInferMNNVLCuteDSLARFusion: dist.barrier(group=process_group) def supports(self, m: int) -> bool: - if self._destroyed or not 1 <= int(m) <= self.max_m: + if not 1 <= int(m) <= self.max_m: return False return self.workspace.is_buffer_size_sufficient( - tp_size=dist.get_world_size(self.process_group), + tp_size=self.tp_size, num_tokens=int(m), hidden_dim=self.hidden_size, dtype=torch.bfloat16, @@ -184,27 +425,20 @@ class FlashInferMNNVLCuteDSLARFusion: gated_shared_output: torch.Tensor, residual: torch.Tensor, gamma: torch.Tensor, - norm_output: torch.Tensor | None = None, - residual_output: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: m = int(permuted_indices.shape[0]) if not self.supports(m): raise ValueError(f"workspace does not support M={m}") shape = (m, self.hidden_size) - if norm_output is None: - norm_output = torch.empty(shape, dtype=torch.bfloat16, device=self.device) - if residual_output is None: - residual_output = torch.empty( - shape, dtype=torch.bfloat16, device=self.device - ) + norm_output = torch.empty(shape, dtype=torch.bfloat16, device=self.device) + residual_output = torch.empty(shape, dtype=torch.bfloat16, device=self.device) pattern = self._patterns.kMoEFinalizeARResidualRMSNorm self._allreduce_fusion( input=routed_output, workspace=self.workspace, pattern=pattern, - # The public API carries the caller's PDL intent. The backend's - # routing profile owns the compiled choice and validates it. + # Caller intent only; the routing profile owns the compiled choice. launch_with_pdl=True, residual_in=residual, residual_out=residual_output, @@ -224,16 +458,12 @@ class FlashInferMNNVLCuteDSLARFusion: local_contribution: torch.Tensor, residual: torch.Tensor, gamma: torch.Tensor, - norm_output: torch.Tensor | None = None, - residual_output: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: m = int(local_contribution.shape[0]) if not self.supports(m): raise ValueError(f"workspace does not support M={m}") - if norm_output is None: - norm_output = torch.empty_like(local_contribution) - if residual_output is None: - residual_output = torch.empty_like(local_contribution) + norm_output = torch.empty_like(local_contribution) + residual_output = torch.empty_like(local_contribution) pattern = self._patterns.kARResidualRMSNorm self._allreduce_fusion( @@ -250,124 +480,43 @@ class FlashInferMNNVLCuteDSLARFusion: ) return norm_output, residual_output - def destroy(self) -> None: - if self._destroyed: - return - self.workspace.destroy() - self._destroyed = True - -_WORKSPACES: dict[_WorkspaceSignature, FlashInferMNNVLCuteDSLARFusion] = {} -_WORKSPACES_LOCK = threading.RLock() - - -@lru_cache(maxsize=1) -def _max_workspace_instances() -> int: - from sglang.srt.environ import envs - - value = int(envs.SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION_MAX_INSTANCES.get()) - if value < 1: - raise ValueError( - "SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION_MAX_INSTANCES must be positive" - ) - return value +_WORKSPACE: FlashInferMNNVLCuteDSLARFusion | None = None def get_flashinfer_mnnvl_cutedsl_ar_fusion( *, - hidden_size: int | None = None, - top_k: int | None = None, - max_m: int | None = None, - rms_epsilon: float | None = None, - weight_bias: float | None = None, + hidden_size: int, + top_k: int, + max_m: int, + rms_epsilon: float, + weight_bias: float, ) -> FlashInferMNNVLCuteDSLARFusion: - """Lookup, or before graph capture create, the process-local workspace.""" - supplied = (hidden_size, top_k, max_m, rms_epsilon, weight_bias) - if all(value is None for value in supplied): - with _WORKSPACES_LOCK: - if len(_WORKSPACES) == 1: - return next(iter(_WORKSPACES.values())) - if not _WORKSPACES: - raise RuntimeError( - "MNNVL CuTe DSL fusion workspace was not initialized before use" - ) - raise RuntimeError( - "multiple MNNVL CuTe DSL fusion workspaces exist; configuration " - "arguments are required" - ) - if any(value is None for value in supplied): - raise TypeError( - "hidden_size, top_k, max_m, rms_epsilon, and weight_bias must be " - "supplied together" + """Build the process-local workspace. Must run before graph capture.""" + global _WORKSPACE + if _WORKSPACE is not None: + raise RuntimeError( + "a second MNNVL CuTe DSL fusion workspace was requested; each one " + "rendezvouses its own NVLS region, and a process serves one model" ) if not torch.cuda.is_available(): raise RuntimeError("MNNVL CuTe DSL fusion requires CUDA") - assert hidden_size is not None - assert top_k is not None - assert max_m is not None - assert rms_epsilon is not None - assert weight_bias is not None from sglang.srt.runtime_context import get_parallel - device = torch.device("cuda", torch.cuda.current_device()) - process_group = get_parallel().tp_group.device_group - domain = ( - int(hidden_size), - int(top_k), - float(rms_epsilon), - float(weight_bias), - int(device.index), - id(process_group), + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError( + "creating an MNNVL CuTe DSL fusion workspace during CUDA Graph " + "capture is forbidden" + ) + + _WORKSPACE = FlashInferMNNVLCuteDSLARFusion( + hidden_size=hidden_size, + top_k=top_k, + max_m=max_m, + rms_epsilon=rms_epsilon, + weight_bias=weight_bias, + process_group=get_parallel().tp_group.device_group, + device=torch.device("cuda", torch.cuda.current_device()), ) - - with _WORKSPACES_LOCK: - compatible = [ - (signature.max_m, instance) - for signature, instance in _WORKSPACES.items() - if ( - signature.hidden_size, - signature.top_k, - signature.rms_epsilon, - signature.weight_bias, - signature.device_index, - signature.process_group_identity, - ) - == domain - and signature.max_m >= int(max_m) - ] - if compatible: - return min(compatible, key=lambda item: item[0])[1] - - if torch.cuda.is_current_stream_capturing(): - raise RuntimeError( - "creating an MNNVL CuTe DSL fusion workspace during CUDA Graph " - "capture is forbidden" - ) - if len(_WORKSPACES) >= _max_workspace_instances(): - raise RuntimeError( - "MNNVL CuTe DSL fusion workspace instance limit exceeded; " - "increase SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION_MAX_INSTANCES " - "only when multiple model configurations intentionally coexist" - ) - - signature = _WorkspaceSignature( - hidden_size=int(hidden_size), - top_k=int(top_k), - rms_epsilon=float(rms_epsilon), - weight_bias=float(weight_bias), - max_m=int(max_m), - device_index=int(device.index), - process_group_identity=id(process_group), - ) - instance = FlashInferMNNVLCuteDSLARFusion( - hidden_size=hidden_size, - top_k=top_k, - max_m=max_m, - rms_epsilon=rms_epsilon, - weight_bias=weight_bias, - process_group=process_group, - device=device, - ) - _WORKSPACES[signature] = instance - return instance + return _WORKSPACE diff --git a/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py b/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py new file mode 100644 index 0000000000..9b7ab4e08c --- /dev/null +++ b/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py @@ -0,0 +1,478 @@ +"""FlashInfer MNNVL CuTe DSL AllReduce fusion, shared across architectures. + +Two patterns share one workspace, both consumed at the next layer's input +RMSNorm: AR + residual + RMSNorm, and the same with the MoE finalize and the +shared-expert add folded in when the runner hands back a MoeFinalizeHandoff. +""" + +from __future__ import annotations + +import functools +import logging +from typing import Callable, Optional, Sequence + +import msgspec +import torch + +from sglang.srt.layers.communicator import ( + CommunicateWithAllReduceAndLayerNormFn, + LayerCommunicator, + ScatterMode, + get_attn_tp_context, +) +from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm +from sglang.srt.layers.moe import get_moe_a2a_backend +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.runtime_context import ( + cutedsl_moe_max_num_tokens, + get_disagg, + get_exec, + get_flags, + get_parallel, +) + +_LayerPredicate = Callable[[torch.nn.Module], bool] + +logger = logging.getLogger(__name__) + + +def _fused_norm_gamma(layernorm: torch.nn.Module) -> Optional[torch.Tensor]: + """The multiplier as applied -- GemmaRMSNorm's pre-folded w + 1. None declines.""" + if isinstance(layernorm, GemmaRMSNorm): + return layernorm.gemma_weight + if isinstance(layernorm, RMSNorm) and layernorm.has_weight: + return layernorm.weight + return None + + +def _is_supported_forward_mode(forward_mode: ForwardMode) -> bool: + return forward_mode in ( + ForwardMode.DECODE, + ForwardMode.EXTEND, + ForwardMode.TARGET_VERIFY, + ) + + +def _resolve_max_m(*, max_running_requests: int | None) -> int: + decode_config = get_exec().graph.cuda_graph_config.decode + prefill_config = get_exec().graph.cuda_graph_config.prefill + candidates = [ + cutedsl_moe_max_num_tokens(), + max_running_requests, + decode_config.max_bs, + prefill_config.max_bs, + *(decode_config.bs or []), + *(prefill_config.bs or []), + ] + positive = [ + int(value) for value in candidates if value is not None and int(value) > 0 + ] + if not positive: + raise RuntimeError("framework reported no positive fusion workspace M bound") + return max(positive) + + +class MoeFinalizeHandoff(msgspec.Struct, frozen=True): + """Unfinalized routed output plus the separately gated shared contribution.""" + + routed_output: torch.Tensor + expert_weights: torch.Tensor + permuted_indices: torch.Tensor + gated_shared_output: torch.Tensor + m: int + + @classmethod + def from_flashinfer( + cls, + deferred_output, + *, + gated_shared_output: torch.Tensor, + m: int, + ) -> MoeFinalizeHandoff: + top_k = int(deferred_output.top_k) + return cls( + routed_output=deferred_output.gemm2_out.view( + -1, deferred_output.gemm2_out.shape[-1] + ), + expert_weights=deferred_output.expert_weights.view(-1, top_k)[:m], + permuted_indices=deferred_output.expanded_idx_to_permuted_idx.view( + -1, top_k + )[:m], + gated_shared_output=gated_shared_output, + m=int(m), + ) + + +class CuteDSLFusionService: + def __init__( + self, + *, + hidden_size: int, + top_k: int, + rms_epsilon: float, + ) -> None: + self.hidden_size = int(hidden_size) + self.top_k = int(top_k) + self.rms_epsilon = float(rms_epsilon) + self.max_m: int | None = None + self._workspace = None + + def prepare(self, *, max_m: int) -> None: + if self._workspace is not None: + assert self.max_m is not None + if int(max_m) > self.max_m: + raise RuntimeError( + f"fusion workspace is already prepared for M_max={self.max_m}; " + f"refusing M_max={max_m}" + ) + return + from sglang.srt.layers.flashinfer_mnnvl_cutedsl import ( + get_flashinfer_mnnvl_cutedsl_ar_fusion, + ) + + workspace = get_flashinfer_mnnvl_cutedsl_ar_fusion( + hidden_size=self.hidden_size, + top_k=self.top_k, + max_m=int(max_m), + rms_epsilon=self.rms_epsilon, + # _fused_norm_gamma() already returns the multiplier as applied. + weight_bias=0.0, + ) + self._workspace = workspace + self.max_m = workspace.max_m + + def supports(self, m: int) -> bool: + """False before prepare(), so it doubles as the readiness check.""" + return self._workspace is not None and self._workspace.supports(m) + + def finalize( + self, + handoff: MoeFinalizeHandoff, + residual: torch.Tensor, + gamma: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + assert self._workspace is not None + return self._workspace.moe_finalize_all_reduce_rms_norm( + routed_output=handoff.routed_output, + expert_weights=handoff.expert_weights, + permuted_indices=handoff.permuted_indices, + gated_shared_output=handoff.gated_shared_output, + residual=residual, + gamma=gamma, + ) + + def all_reduce_residual_rms_norm( + self, + local_contribution: torch.Tensor, + residual: torch.Tensor, + gamma: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + assert self._workspace is not None + return self._workspace.all_reduce_residual_rms_norm( + local_contribution=local_contribution, + residual=residual, + gamma=gamma, + ) + + +class CuteDSLFusionLayerCommunicator(LayerCommunicator): + fusion_service: CuteDSLFusionService | None = None + + # The runner can defer and a successor or the final norm consumes the handoff. + may_defer_moe_finalize: bool = False + # False on the last layer: the final norm does not all-reduce. + successor_absorbs_all_reduce: bool = False + # A replicated output follows the reduction; moving it would scale that by tp. + owes_local_reduction: bool = False + + def prepare_attn( + self, + hidden_states, + residual, + forward_batch, + quant_format: str = "", + post_residual_addition=None, + ): + if isinstance(hidden_states, MoeFinalizeHandoff): + if not self._should_use_finalize(forward_batch, hidden_states.m): + raise RuntimeError( + "received deferred MoE output on an ineligible path " + f"(M={hidden_states.m}, mode={forward_batch.forward_mode})" + ) + if residual is None: + raise RuntimeError("deferred MoE finalize requires residual input") + gamma = _fused_norm_gamma(self.input_layernorm) + if gamma is None: + raise RuntimeError( + "deferred MoE finalize requires a fusable RMSNorm flavour" + ) + if post_residual_addition is not None: + residual = residual + post_residual_addition + assert self.fusion_service is not None + hidden_states, residual = self.fusion_service.finalize( + handoff=hidden_states, residual=residual, gamma=gamma + ) + return self._finish_prepare_attn( + hidden_states=hidden_states, + residual=residual, + forward_batch=forward_batch, + ) + + if ( + residual is not None + and hasattr(hidden_states, "_sglang_needs_allreduce_fusion") + and hidden_states._sglang_needs_allreduce_fusion + and self._can_consume_post_moe_all_reduce( + forward_batch, int(hidden_states.shape[0]) + ) + ): + if post_residual_addition is not None: + residual = residual + post_residual_addition + assert self.fusion_service is not None + hidden_states, residual = self.fusion_service.all_reduce_residual_rms_norm( + local_contribution=hidden_states, + residual=residual, + gamma=_fused_norm_gamma(self.input_layernorm), + ) + return self._finish_prepare_attn( + hidden_states=hidden_states, + residual=residual, + forward_batch=forward_batch, + ) + + return super().prepare_attn( + hidden_states, + residual, + forward_batch, + quant_format=quant_format, + post_residual_addition=post_residual_addition, + ) + + def prepare_mlp( + self, + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + cache=None, + ): + if cache is not None: + self._context.cache = cache + if self._should_use_all_reduce_rms_norm( + forward_batch, int(hidden_states.shape[0]), residual + ): + assert self.fusion_service is not None and residual is not None + return self.fusion_service.all_reduce_residual_rms_norm( + local_contribution=hidden_states, + residual=residual, + gamma=_fused_norm_gamma(self.post_attention_layernorm), + ) + return super().prepare_mlp(hidden_states, residual, forward_batch, cache=cache) + + def _should_use_all_reduce_rms_norm( + self, + forward_batch: ForwardBatch, + m: int, + residual: Optional[torch.Tensor], + ) -> bool: + communicate_fn = self._communicate_with_all_reduce_and_layer_norm_fn + if isinstance(communicate_fn, functools.partial): + norm_fn = communicate_fn.func + residual_input_mode = communicate_fn.keywords.get("residual_input_mode") + else: + norm_fn = communicate_fn + residual_input_mode = None + parallel = get_parallel() + return ( + self._common_eligible(forward_batch, m) + and residual is not None + and _fused_norm_gamma(self.post_attention_layernorm) is not None + and norm_fn + is CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual + and residual_input_mode is ScatterMode.TP_ATTN_FULL + and self._context.attn_dp_size == 1 + and parallel.attn_tp_size == parallel.tp_size + and not get_exec().comm.enable_quant_communications + ) + + def _should_use_finalize(self, forward_batch: ForwardBatch, m: int) -> bool: + return ( + self._common_eligible(forward_batch, m) and get_parallel().moe_ep_size == 1 + ) + + def _can_consume_post_moe_all_reduce( + self, forward_batch: ForwardBatch, m: int + ) -> bool: + """Incoming, and independent of this layer's own successor.""" + return ( + self._common_eligible(forward_batch, m) + and _fused_norm_gamma(self.input_layernorm) is not None + and not get_exec().comm.enable_quant_communications + ) + + def _can_absorb_post_moe_all_reduce( + self, forward_batch: ForwardBatch, m: int + ) -> bool: + """Outgoing: skip our own all-reduce because the next layer absorbs it.""" + return ( + self.successor_absorbs_all_reduce + and not self.owes_local_reduction + and self._can_consume_post_moe_all_reduce(forward_batch, m) + ) + + def should_defer_moe_finalize( + self, forward_batch: ForwardBatch, m: int | None = None + ) -> bool: + """Deferring skips the post-experts all-reduce on the promise of a handoff.""" + if not self.may_defer_moe_finalize: + return False + if m is None: + m = int(forward_batch.input_ids.shape[0]) + return self._should_use_finalize(forward_batch, m) + + def _common_eligible(self, forward_batch: ForwardBatch, m: int) -> bool: + parallel = get_parallel() + return bool( + self.fusion_service is not None + and _is_supported_forward_mode(forward_batch.forward_mode) + and self.fusion_service.supports(m) + and not is_dp_attention_enabled() + # Also forces moe_dp_size == 1, so a hybrid EP x MoE-TP reduction + # always merges into the one TP reduction the workspace performs. + and parallel.attn_cp_size == 1 + and not get_attn_tp_context().input_scattered + and get_moe_a2a_backend().is_none() + and self._context.tp_size > 1 + # Restates the base's moe-cp, MOE_FULL and SCATTERED refusals. + and self.layer_scatter_modes.mlp_mode is ScatterMode.FULL + ) + + def should_fuse_mlp_allreduce_with_next_layer( + self, forward_batch: ForwardBatch + ) -> bool: + m = int(forward_batch.input_ids.shape[0]) + if self.should_defer_moe_finalize(forward_batch, m): + return True + if self._can_absorb_post_moe_all_reduce(forward_batch, m): + return True + return super().should_fuse_mlp_allreduce_with_next_layer(forward_batch) + + +def install_cutedsl_fusion( + layers: Sequence[torch.nn.Module], + *, + hidden_size: int, + top_k: int, + rms_epsilon: float, + can_defer_finalize: _LayerPredicate, + requires_local_reduction: _LayerPredicate | None = None, + final_norm_consumes_handoff: bool = False, + label: str, +) -> CuteDSLFusionService | None: + """One shared workspace handle per fusion-enabled layer, or None. + + Every entry of ``layers`` must carry a ``layer_communicator``. + """ + if get_flags().moe.in_speculative_scope: + # A draft shares the target's process, which holds one workspace. + return None + + fusion_layers = [ + layer + for layer in layers + if isinstance(layer.layer_communicator, CuteDSLFusionLayerCommunicator) + ] + if not fusion_layers: + return None + + # The workspace compiles one epsilon and cannot see a per-layer one. + for layer in fusion_layers: + for norm in ( + layer.layer_communicator.input_layernorm, + layer.layer_communicator.post_attention_layernorm, + ): + if _fused_norm_gamma(norm) is None: + continue + if float(norm.variance_epsilon) != float(rms_epsilon): + raise RuntimeError( + f"{label} CuTe DSL fusion compiles one workspace for " + f"rms_epsilon={rms_epsilon}, but a fused norm uses " + f"{norm.variance_epsilon}" + ) + + service = CuteDSLFusionService( + hidden_size=hidden_size, + top_k=top_k, + rms_epsilon=rms_epsilon, + ) + for index, layer in enumerate(layers): + communicator = layer.layer_communicator + if not isinstance(communicator, CuteDSLFusionLayerCommunicator): + continue + successor = layers[index + 1] if index + 1 < len(layers) else None + if successor is None: + has_consumer = final_norm_consumes_handoff + else: + has_consumer = isinstance( + successor.layer_communicator, CuteDSLFusionLayerCommunicator + ) + communicator.fusion_service = service + communicator.may_defer_moe_finalize = ( + bool(can_defer_finalize(layer)) and has_consumer + ) + communicator.successor_absorbs_all_reduce = successor is not None and ( + isinstance(successor.layer_communicator, CuteDSLFusionLayerCommunicator) + ) + communicator.owes_local_reduction = ( + requires_local_reduction is not None and requires_local_reduction(layer) + ) + logger.info( + "Installed one %s FlashInfer MNNVL CuTe DSL fusion handle for %d of %d layers " + "(%d can defer the MoE finalize)", + label, + len(fusion_layers), + len(layers), + sum(layer.layer_communicator.may_defer_moe_finalize for layer in fusion_layers), + ) + return service + + +def prepare_cutedsl_fusion( + model: torch.nn.Module, *, max_running_requests: int | None +) -> None: + """Build the workspace of every service installed anywhere in ``model``. + + Scans the module tree, so a wrapper holding the model as a submodule (a VLM's + language_model) needs no hook of its own. + """ + communicators = [ + communicator + for module in model.modules() + # Most modules carry no layer_communicator. + if isinstance( + communicator := module.__dict__.get("layer_communicator"), + CuteDSLFusionLayerCommunicator, + ) + ] + if not communicators or any(c.fusion_service is None for c in communicators): + raise ValueError( + "--flashinfer-allreduce-fusion-backend cutedsl is set, but " + f"{type(model).__name__} installed no CuTe DSL fusion service, so no " + "allreduce fusion would run at all. Drop the flag, or choose 'auto', " + "'trtllm' or 'mnnvl'." + ) + if get_disagg().enable_pdmux: + raise RuntimeError( + "FlashInfer MNNVL CuTe DSL fusion does not support concurrent PDMux " + "streams sharing one mutable workspace" + ) + services = {id(c.fusion_service): c.fusion_service for c in communicators} + max_m = _resolve_max_m(max_running_requests=max_running_requests) + for service in services.values(): + service.prepare(max_m=max_m) + logger.info( + "Prepared the FlashInfer MNNVL CuTe DSL fusion workspace for M_max=%d " + "(%d fused layers)", + max_m, + len(communicators), + ) diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py index 95ba2c4b3a..5895f2c786 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py @@ -1495,13 +1495,7 @@ def fused_experts_none_to_flashinfer_trtllm_fp4( result = trtllm_fp4_block_scale_moe(**moe_kwargs) if defer_finalize: - gemm2_out, expert_weights, expanded_idx_to_permuted_idx = result[:3] - result = FlashInferTrtllmDeferredFinalizeOutput( - gemm2_out=gemm2_out, - expert_weights=expert_weights, - expanded_idx_to_permuted_idx=expanded_idx_to_permuted_idx, - top_k=topk_config.top_k, - ) + result = _make_deferred_finalize_output(result, top_k=topk_config.top_k) else: result = result[0] diff --git a/python/sglang/srt/layers/moe/qwen35_flashinfer_fusion.py b/python/sglang/srt/layers/moe/qwen35_flashinfer_fusion.py deleted file mode 100644 index 606676a033..0000000000 --- a/python/sglang/srt/layers/moe/qwen35_flashinfer_fusion.py +++ /dev/null @@ -1,350 +0,0 @@ -"""Qwen3.5 integration for FlashInfer MNNVL CuTe DSL AllReduce fusion.""" - -from __future__ import annotations - -import logging -from dataclasses import dataclass -from typing import Optional - -import torch - -from sglang.srt.arg_groups.overrides import cutedsl_moe_max_num_tokens -from sglang.srt.layers.communicator import ( - CommunicateWithAllReduceAndLayerNormFn, - LayerCommunicator, - ScatterMode, - get_attn_tp_context, -) -from sglang.srt.layers.dp_attention import is_dp_attention_enabled -from sglang.srt.layers.moe import get_moe_a2a_backend -from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode -from sglang.srt.runtime_context import ( - get_disagg, - get_exec, - get_parallel, -) - -logger = logging.getLogger(__name__) - - -def is_supported_forward_mode(forward_mode: ForwardMode) -> bool: - return forward_mode in ( - ForwardMode.DECODE, - ForwardMode.EXTEND, - ForwardMode.TARGET_VERIFY, - ) - - -def resolve_max_m(model_runner) -> int: - """Use framework token bounds as the workspace-capacity source of truth.""" - decode_config = get_exec().graph.cuda_graph_config.decode - prefill_config = get_exec().graph.cuda_graph_config.prefill - candidates = [ - cutedsl_moe_max_num_tokens(model_runner.server_args), - model_runner.max_running_requests, - decode_config.max_bs, - prefill_config.max_bs, - *(decode_config.bs or []), - *(prefill_config.bs or []), - ] - positive = [ - int(value) for value in candidates if value is not None and int(value) > 0 - ] - if not positive: - raise RuntimeError("framework reported no positive fusion workspace M bound") - return max(positive) - - -@dataclass(frozen=True) -class Qwen35MoeFinalizeHandoff: - """Unfinalized routed output plus the separately gated shared contribution.""" - - routed_output: torch.Tensor - expert_weights: torch.Tensor - permuted_indices: torch.Tensor - gated_shared_output: torch.Tensor - m: int - - @classmethod - def from_flashinfer( - cls, - deferred_output, - *, - gated_shared_output: torch.Tensor, - m: int, - ) -> Qwen35MoeFinalizeHandoff: - top_k = int(deferred_output.top_k) - return cls( - routed_output=deferred_output.gemm2_out.view( - -1, deferred_output.gemm2_out.shape[-1] - ), - expert_weights=deferred_output.expert_weights.view(-1, top_k)[:m], - permuted_indices=deferred_output.expanded_idx_to_permuted_idx.view( - -1, top_k - )[:m], - gated_shared_output=gated_shared_output, - m=int(m), - ) - - -class Qwen35FlashInferFusionService: - """A lightweight model handle for the process-local FlashInfer workspace.""" - - def __init__( - self, - *, - hidden_size: int, - top_k: int, - rms_epsilon: float, - ) -> None: - self.hidden_size = int(hidden_size) - self.top_k = int(top_k) - self.rms_epsilon = float(rms_epsilon) - self.max_m: int | None = None - self._workspace = None - - @property - def is_prepared(self) -> bool: - return self._workspace is not None - - def prepare(self, *, max_m: int) -> None: - if self._workspace is not None: - assert self.max_m is not None - if int(max_m) > self.max_m: - raise RuntimeError( - f"fusion workspace is already prepared for M_max={self.max_m}; " - f"refusing M_max={max_m}" - ) - return - from sglang.srt.layers.flashinfer_mnnvl_cutedsl import ( - get_flashinfer_mnnvl_cutedsl_ar_fusion, - ) - - workspace = get_flashinfer_mnnvl_cutedsl_ar_fusion( - hidden_size=self.hidden_size, - top_k=self.top_k, - max_m=int(max_m), - rms_epsilon=self.rms_epsilon, - # GemmaRMSNorm.gemma_weight is already checkpoint weight + 1. - weight_bias=0.0, - ) - self._workspace = workspace - self.max_m = workspace.max_m - - def supports(self, m: int) -> bool: - if self._workspace is None or self.max_m is None: - return False - return 1 <= int(m) <= self.max_m and self._workspace.supports(m) - - def finalize( - self, - handoff: Qwen35MoeFinalizeHandoff, - residual: torch.Tensor, - gamma: torch.Tensor, - ) -> tuple[torch.Tensor, torch.Tensor]: - self._validate_finalize(handoff, residual, gamma) - assert self._workspace is not None - return self._workspace.moe_finalize_all_reduce_rms_norm( - routed_output=handoff.routed_output, - expert_weights=handoff.expert_weights, - permuted_indices=handoff.permuted_indices, - gated_shared_output=handoff.gated_shared_output, - residual=residual, - gamma=gamma, - ) - - def all_reduce_residual_rms_norm( - self, - local_contribution: torch.Tensor, - residual: torch.Tensor, - gamma: torch.Tensor, - ) -> tuple[torch.Tensor, torch.Tensor]: - self._validate_matrix(local_contribution, "local_contribution") - self._validate_matrix(residual, "residual", m=local_contribution.shape[0]) - self._validate_gamma(gamma) - if not self.supports(local_contribution.shape[0]): - raise ValueError(f"unsupported M={local_contribution.shape[0]}") - assert self._workspace is not None - return self._workspace.all_reduce_residual_rms_norm( - local_contribution=local_contribution, - residual=residual, - gamma=gamma, - ) - - def _validate_finalize( - self, - handoff: Qwen35MoeFinalizeHandoff, - residual: torch.Tensor, - gamma: torch.Tensor, - ) -> None: - if not self.supports(handoff.m): - raise ValueError(f"unsupported M={handoff.m}") - self._validate_matrix(handoff.routed_output, "routed_output", exact_m=False) - expected_metadata = (handoff.m, self.top_k) - if tuple(handoff.expert_weights.shape) != expected_metadata: - raise ValueError("expert_weights must have shape [M, top_k]") - if tuple(handoff.permuted_indices.shape) != expected_metadata: - raise ValueError("permuted_indices must have shape [M, top_k]") - if handoff.expert_weights.dtype != torch.bfloat16: - raise ValueError("expert_weights must be BF16") - if handoff.permuted_indices.dtype != torch.int32: - raise ValueError("permuted_indices must be Int32") - self._validate_matrix( - handoff.gated_shared_output, "gated_shared_output", m=handoff.m - ) - self._validate_matrix(residual, "residual", m=handoff.m) - self._validate_gamma(gamma) - - def _validate_matrix( - self, - tensor: torch.Tensor, - name: str, - *, - m: int | None = None, - exact_m: bool = True, - ) -> None: - if tensor.ndim != 2 or tensor.shape[1] != self.hidden_size: - raise ValueError(f"{name} must have shape [M, hidden_size]") - if m is not None and exact_m and tensor.shape[0] != int(m): - raise ValueError(f"{name} has the wrong M dimension") - if tensor.dtype != torch.bfloat16 or not tensor.is_contiguous(): - raise ValueError(f"{name} must be contiguous BF16") - - def _validate_gamma(self, gamma: torch.Tensor) -> None: - if ( - tuple(gamma.shape) != (self.hidden_size,) - or gamma.dtype != torch.bfloat16 - or not gamma.is_contiguous() - ): - raise ValueError("gamma must be contiguous BF16 [hidden_size]") - - -class Qwen35FlashInferLayerCommunicator(LayerCommunicator): - """Qwen-only hooks; generic LayerCommunicator remains backend agnostic.""" - - fusion_service: Qwen35FlashInferFusionService | None = None - - def prepare_attn( - self, - hidden_states, - residual, - forward_batch, - quant_format: str = "", - post_residual_addition=None, - ): - if isinstance(hidden_states, Qwen35MoeFinalizeHandoff): - if not self.should_use_finalize(forward_batch, hidden_states.m): - raise RuntimeError("received deferred MoE output on an ineligible path") - if residual is None: - raise RuntimeError("deferred MoE finalize requires residual input") - if not hasattr(self.input_layernorm, "gemma_weight"): - raise RuntimeError("deferred Qwen finalize requires GemmaRMSNorm") - if post_residual_addition is not None: - residual = residual + post_residual_addition - assert self.fusion_service is not None - return self.fusion_service.finalize( - hidden_states, residual, self.input_layernorm.gemma_weight - ) - return super().prepare_attn( - hidden_states, - residual, - forward_batch, - quant_format=quant_format, - post_residual_addition=post_residual_addition, - ) - - def prepare_mlp( - self, - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - cache=None, - ): - if cache is not None: - self._context.cache = cache - if self.should_use_all_reduce_rms_norm( - forward_batch, int(hidden_states.shape[0]), residual - ): - assert self.fusion_service is not None and residual is not None - return self.fusion_service.all_reduce_residual_rms_norm( - hidden_states, - residual, - self.post_attention_layernorm.gemma_weight, - ) - return super().prepare_mlp(hidden_states, residual, forward_batch, cache=cache) - - def should_use_all_reduce_rms_norm( - self, - forward_batch: ForwardBatch, - m: int, - residual: Optional[torch.Tensor], - ) -> bool: - communicate_fn = self._communicate_with_all_reduce_and_layer_norm_fn - norm_fn = getattr(communicate_fn, "func", communicate_fn) - residual_input_mode = getattr(communicate_fn, "keywords", {}).get( - "residual_input_mode" - ) - parallel = get_parallel() - return ( - self._common_eligible(forward_batch, m) - and residual is not None - and hasattr(self.post_attention_layernorm, "gemma_weight") - and norm_fn - is CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual - and residual_input_mode is ScatterMode.TP_ATTN_FULL - and self._context.attn_dp_size == 1 - and parallel.attn_tp_size == parallel.tp_size - and not get_exec().comm.enable_quant_communications - ) - - def should_use_finalize(self, forward_batch: ForwardBatch, m: int) -> bool: - parallel = get_parallel() - return ( - self._common_eligible(forward_batch, m) - and self.layer_scatter_modes.mlp_mode is not ScatterMode.SCATTERED - and parallel.moe_ep_size == 1 - ) - - def _common_eligible(self, forward_batch: ForwardBatch, m: int) -> bool: - parallel = get_parallel() - return bool( - self.fusion_service is not None - and self.fusion_service.is_prepared - and is_supported_forward_mode(forward_batch.forward_mode) - and self.fusion_service.supports(m) - and not is_dp_attention_enabled() - and parallel.attn_cp_size == 1 - and not get_attn_tp_context().input_scattered - and get_moe_a2a_backend().is_none() - and self._context.tp_size > 1 - ) - - def should_fuse_mlp_allreduce_with_next_layer( - self, forward_batch: ForwardBatch - ) -> bool: - m = ( - int(forward_batch.input_ids.shape[0]) - if getattr(forward_batch, "input_ids", None) is not None - else 0 - ) - if self.should_use_finalize(forward_batch, m): - # The Qwen model consumes the final layer's handoff with its final - # GemmaRMSNorm, so this is intentionally also true for that layer. - return True - return super().should_fuse_mlp_allreduce_with_next_layer(forward_batch) - - -def prepare_qwen35_flashinfer_fusion(model, model_runner) -> None: - service = getattr(model, "flashinfer_mnnvl_cutedsl_fusion", None) - if service is None: - return - if get_disagg().enable_pdmux: - raise RuntimeError( - "FlashInfer MNNVL CuTe DSL fusion does not support concurrent PDMux " - "streams sharing one mutable workspace" - ) - service.prepare(max_m=resolve_max_m(model_runner)) - logger.info( - "Prepared Qwen3.5 FlashInfer MNNVL CuTe DSL fusion workspace for M_max=%d", - service.max_m, - ) diff --git a/python/sglang/srt/model_executor/runner/base_runner.py b/python/sglang/srt/model_executor/runner/base_runner.py index 1c3084f635..7f69e36de5 100644 --- a/python/sglang/srt/model_executor/runner/base_runner.py +++ b/python/sglang/srt/model_executor/runner/base_runner.py @@ -288,11 +288,29 @@ class BaseRunner(ABC): with custom_all_reduce.register_graph_buffers). """ mr = self.model_runner + from sglang.srt.layers.communicator import FUSE_ALLREDUCE_MAX_BATCH_SIZE + from sglang.srt.layers.flashinfer_comm_fusion import ( + pre_initialize_workspaces, + resolve_flashinfer_allreduce_fusion_backend, + uses_cutedsl_ar_fusion, + ) + if get_exec().comm.flashinfer_allreduce_fusion_backend is None: return - from sglang.srt.layers.communicator import FUSE_ALLREDUCE_MAX_BATCH_SIZE - from sglang.srt.layers.flashinfer_comm_fusion import pre_initialize_workspaces + if uses_cutedsl_ar_fusion(): + # Nothing else resolves the configured backend, so check the platform. + resolve_flashinfer_allreduce_fusion_backend() + if not mr.is_draft_worker: + # A draft installs no fusion communicator. + from sglang.srt.layers.moe.cutedsl_ar_fusion import ( + prepare_cutedsl_fusion, + ) + + prepare_cutedsl_fusion( + mr.model, max_running_requests=mr.max_running_requests + ) + return pre_initialize_workspaces( max_token_num=FUSE_ALLREDUCE_MAX_BATCH_SIZE, diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 9cbc9dbd70..b8c0177e48 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -81,6 +81,7 @@ from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx from sglang.srt.layers.dcp.planner import ( prepare_decode_context_parallel_metadata, ) +from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, @@ -575,6 +576,9 @@ class DeepseekV2MoE(nn.Module): self.moe_ep_size = get_parallel().moe_ep_size self.routed_scaling_factor = config.routed_scaling_factor self.n_shared_experts = config.n_shared_experts + self._deferred_finalize_max_tokens = ( + envs.SGLANG_MOE_DEFERRED_FINALIZE_MAX_TOKENS.get() + ) n_shared_experts = ( 0 if config.n_shared_experts is None else int(config.n_shared_experts) @@ -1061,6 +1065,10 @@ class DeepseekV2MoE(nn.Module): and not self._shared_expert_tp1 and topk_output.format == TopKOutputFormat.BYPASSED and self.experts.supports_deferred_finalize + and ( + self._deferred_finalize_max_tokens <= 0 + or hidden_states.shape[0] <= self._deferred_finalize_max_tokens + ) ) if deferred_finalize: final_hidden_states = self.experts.forward_deferred_finalize( @@ -1094,6 +1102,17 @@ class DeepseekV2MoE(nn.Module): # The routed-input pre-quant was already joined inside the routed MoE apply. current_stream.wait_stream(self.alt_stream) + if deferred_finalize and get_forward().defer_moe_finalize: + # deferred_finalize excludes _shared_expert_tp1, so the shared add folds in. + assert shared_output is not None + from sglang.srt.layers.moe.cutedsl_ar_fusion import MoeFinalizeHandoff + + return MoeFinalizeHandoff.from_flashinfer( + final_hidden_states, + gated_shared_output=shared_output, + m=hidden_states.shape[0], + ) + all_reduce_done = False if deferred_finalize: from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( @@ -2482,6 +2501,10 @@ class DeepseekV2AttentionMLA( return quant_config +def _use_mnnvl_cutedsl_fusion() -> bool: + return _is_cuda and uses_cutedsl_ar_fusion() + + class DeepseekV2DecoderLayer(nn.Module): def __init__( self, @@ -2586,11 +2609,17 @@ class DeepseekV2DecoderLayer(nn.Module): self._gfx95_quant_format = self._detect_gfx95_quant_format() - communicator_cls = ( - DSACPLayerCommunicator - if get_parallel().enable_prefill_cp - else LayerCommunicator - ) + if get_parallel().enable_prefill_cp: + communicator_cls = DSACPLayerCommunicator + elif not is_nextn and _use_mnnvl_cutedsl_fusion(): + # Dense layers too: selecting cutedsl turns the legacy fusion off. + from sglang.srt.layers.moe.cutedsl_ar_fusion import ( + CuteDSLFusionLayerCommunicator, + ) + + communicator_cls = CuteDSLFusionLayerCommunicator + else: + communicator_cls = LayerCommunicator self.layer_communicator = communicator_cls( layer_scatter_modes=self.layer_scatter_modes, input_layernorm=self.input_layernorm, @@ -2696,6 +2725,8 @@ class DeepseekV2DecoderLayer(nn.Module): isinstance(self.mlp, DeepseekV2MoE) and not self.mlp.experts.moe_runner_config.inplace and not torch.compiler.is_compiling() + # A deferred MoE finalize handoff from the previous layer is no buffer. + and isinstance(hidden_states_orig, torch.Tensor) ): from sglang.srt.layers.moe.moe_runner.base import moe_output_buffer_ctx @@ -2917,6 +2948,34 @@ class DeepseekV2Model(nn.Module): # llama_4_scaling: for supporting Mistral-Large-3 model self.llama_4_scaling_config = getattr(config, "llama_4_scaling", None) + from sglang.srt.layers.moe.cutedsl_ar_fusion import install_cutedsl_fusion + + install_cutedsl_fusion( + # PP pads self.layers with PPMissingLayer, which has no communicator. + # A stage's last layer has no successor, so it keeps its reduction. + self.layers[self.start_layer : self.end_layer], + hidden_size=config.hidden_size, + top_k=config.num_experts_per_tok, + rms_epsilon=config.rms_norm_eps, + # Only the dual-stream path hands off, which needs an unfused shared + # expert; with V3/R1's default shared-experts fusion only AR + norm fuse. + # A TP1 shared expert is added after the all-reduce and cannot move it. + can_defer_finalize=lambda layer: ( + isinstance(layer.mlp, DeepseekV2MoE) + and layer.mlp.experts.supports_deferred_finalize + and layer.mlp.num_fused_shared_experts == 0 + and not layer.mlp._shared_expert_tp1 + ), + # A TP1-replicated output must not ride into the next layer's reduction. + requires_local_reduction=lambda layer: ( + layer.mlp._shared_expert_tp1 + if isinstance(layer.mlp, DeepseekV2MoE) + else layer.mlp.tp_size == 1 + ), + final_norm_consumes_handoff=False, + label="DeepSeek-V3/GLM", + ) + def get_input_embeddings(self) -> torch.Tensor: return self.embed_tokens @@ -3385,6 +3444,8 @@ def dsv2_flashinfer_moe_dual_stream_graph( fuse_mlp_allreduce=fuse_mlp_allreduce, mlp_reduce_scatter=mlp_reduce_scatter, flashinfer_trtllm_bypass=True, + # The op's Tensor schema cannot carry a MoeFinalizeHandoff. + defer_moe_finalize=False, ): return moe_fusion.forward_normal_dual_stream(hidden_states) diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index 692fdafff7..fe5829b2ce 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -51,6 +51,7 @@ from sglang.srt.layers.communicator import ( from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) +from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, @@ -337,8 +338,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module): num_fused_shared_experts=self.num_fused_shared_experts, inplace=not _needs_hidden_after_experts, enable_qwen35_fp8_deferred_finalize=( - config.model_type == "qwen3_5_moe_text" - and envs.SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION.get() + config.model_type == "qwen3_5_moe_text" and uses_cutedsl_ar_fusion() ), ) @@ -865,11 +865,9 @@ class Qwen2MoeSparseMoeBlock(nn.Module): if defer_finalize: if shared_output is None: raise RuntimeError("Qwen deferred finalize requires shared output") - from sglang.srt.layers.moe.qwen35_flashinfer_fusion import ( - Qwen35MoeFinalizeHandoff, - ) + from sglang.srt.layers.moe.cutedsl_ar_fusion import MoeFinalizeHandoff - return Qwen35MoeFinalizeHandoff.from_flashinfer( + return MoeFinalizeHandoff.from_flashinfer( final_hidden_states, gated_shared_output=shared_output, m=num_tokens, diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 798305ee70..9fe49041dd 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -47,6 +47,7 @@ from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) +from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion # Layers - Others from sglang.srt.layers.layernorm import GemmaRMSNorm @@ -172,10 +173,7 @@ def _disable_shared_experts_fusion() -> bool: # intent through the accessor's fallback. # The deferred-finalize ABI needs the shared expert as a separate, gated # local contribution; it cannot consume a shared slot fused into routed MoE. - return bool( - envs.SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION.get() - or is_shared_experts_fusion_disabled() - ) + return bool(uses_cutedsl_ar_fusion() or is_shared_experts_fusion_disabled()) def _maybe_enable_silu_fp4_quant_fusion(mlp: nn.Module) -> None: @@ -209,17 +207,17 @@ def _use_mnnvl_cutedsl_fusion(config: Qwen3_5TextConfig, is_nextn: bool) -> bool return bool( not is_nextn and config.model_type == "qwen3_5_moe_text" - and envs.SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION.get() + and uses_cutedsl_ar_fusion() ) def _layer_communicator_class(config: Qwen3_5TextConfig, is_nextn: bool): if _use_mnnvl_cutedsl_fusion(config, is_nextn): - from sglang.srt.layers.moe.qwen35_flashinfer_fusion import ( - Qwen35FlashInferLayerCommunicator, + from sglang.srt.layers.moe.cutedsl_ar_fusion import ( + CuteDSLFusionLayerCommunicator, ) - return Qwen35FlashInferLayerCommunicator + return CuteDSLFusionLayerCommunicator return LayerCommunicator @@ -305,22 +303,15 @@ def _select_fused_ar_input_for_linear(hidden_states, linear: nn.Module): def _finish_mlp_output(hidden_states, *, expect_deferred: bool): - if not expect_deferred: - if not isinstance(hidden_states, torch.Tensor): - from sglang.srt.layers.moe.qwen35_flashinfer_fusion import ( - Qwen35MoeFinalizeHandoff, - ) + from sglang.srt.layers.moe.cutedsl_ar_fusion import MoeFinalizeHandoff - if isinstance(hidden_states, Qwen35MoeFinalizeHandoff): - raise RuntimeError("unexpected deferred-finalize handoff") + if not expect_deferred: + if isinstance(hidden_states, MoeFinalizeHandoff): + raise RuntimeError("unexpected deferred-finalize handoff") hidden_states._sglang_needs_allreduce_fusion = True return hidden_states - from sglang.srt.layers.moe.qwen35_flashinfer_fusion import ( - Qwen35MoeFinalizeHandoff, - ) - - if not isinstance(hidden_states, Qwen35MoeFinalizeHandoff): + if not isinstance(hidden_states, MoeFinalizeHandoff): raise RuntimeError("Qwen3.5 expected a FlashInfer deferred-finalize handoff") return hidden_states @@ -1199,8 +1190,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module): fuse_mlp_allreduce and isinstance(hidden_states, torch.Tensor) and isinstance(self.mlp, Qwen2MoeSparseMoeBlock) - and hasattr(self.layer_communicator, "should_use_finalize") - and self.layer_communicator.should_use_finalize( + and self.layer_communicator.should_defer_moe_finalize( forward_batch, int(hidden_states.shape[0]) ) ) @@ -1637,8 +1627,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): fuse_mlp_allreduce and isinstance(hidden_states, torch.Tensor) and isinstance(self.mlp, Qwen2MoeSparseMoeBlock) - and hasattr(self.layer_communicator, "should_use_finalize") - and self.layer_communicator.should_use_finalize( + and self.layer_communicator.should_defer_moe_finalize( forward_batch, int(hidden_states.shape[0]) ) ) @@ -1825,26 +1814,20 @@ class Qwen3_5ForCausalLM(nn.Module): "layers: " f"{unsupported_layers}" ) - from sglang.srt.layers.moe.qwen35_flashinfer_fusion import ( - Qwen35FlashInferFusionService, - Qwen35FlashInferLayerCommunicator, + from sglang.srt.layers.moe.cutedsl_ar_fusion import ( + install_cutedsl_fusion, ) - self.flashinfer_mnnvl_cutedsl_fusion = Qwen35FlashInferFusionService( + self.flashinfer_mnnvl_cutedsl_fusion = install_cutedsl_fusion( + self.layers, hidden_size=config.hidden_size, top_k=config.num_experts_per_tok, rms_epsilon=config.rms_norm_eps, - ) - for layer in self.layers: - communicator = layer.layer_communicator - if not isinstance(communicator, Qwen35FlashInferLayerCommunicator): - raise RuntimeError( - "Qwen3.5 fusion-enabled layer has the wrong communicator" - ) - communicator.fusion_service = self.flashinfer_mnnvl_cutedsl_fusion - logger.info( - "Installed one Qwen3.5 FlashInfer fusion handle for %d layers", - len(self.layers), + # Every layer was checked above. + can_defer_finalize=lambda layer: True, + # The final GemmaRMSNorm closes out the last layer's handoff. + final_norm_consumes_handoff=True, + label="Qwen3.5", ) # Final normalization @@ -1879,13 +1862,6 @@ class Qwen3_5ForCausalLM(nn.Module): logger.info( "Packed BF16/FP8 GDN input projection enabled for %d layers", packed ) - if self.flashinfer_mnnvl_cutedsl_fusion is None: - return - from sglang.srt.layers.moe.qwen35_flashinfer_fusion import ( - prepare_qwen35_flashinfer_fusion, - ) - - prepare_qwen35_flashinfer_fusion(self, model_runner) def set_dflash_layers_to_capture(self, layers_to_capture: list[int]): self.layers_to_capture = layers_to_capture @@ -1972,61 +1948,23 @@ class Qwen3_5ForCausalLM(nn.Module): ) # The final layer has no successor to consume its deferred MoE tail. - trace_final_norm = envs.SGLANG_TRACE_QWEN35_FINAL_NORM.get() - use_native_final_norm = envs.SGLANG_QWEN35_NATIVE_FINAL_NORM.get() is_deferred_finalize = False if self.flashinfer_mnnvl_cutedsl_fusion is not None: - from sglang.srt.layers.moe.qwen35_flashinfer_fusion import ( - Qwen35MoeFinalizeHandoff, - ) + from sglang.srt.layers.moe.cutedsl_ar_fusion import MoeFinalizeHandoff - is_deferred_finalize = isinstance(hidden_states, Qwen35MoeFinalizeHandoff) + is_deferred_finalize = isinstance(hidden_states, MoeFinalizeHandoff) if is_deferred_finalize: - if residual is None or self.flashinfer_mnnvl_cutedsl_fusion is None: + if residual is None: raise RuntimeError("invalid final deferred MoE handoff") hidden_states, _ = self.flashinfer_mnnvl_cutedsl_fusion.finalize( - hidden_states, residual, self.norm.gemma_weight + handoff=hidden_states, residual=residual, gamma=self.norm.gemma_weight ) elif hidden_states.shape[0] != 0: - if trace_final_norm: - print( - "SGLANG_TRACE_QWEN35_FINAL_NORM " - f"stage=pre_sync_enter hidden={tuple(hidden_states.shape)} " - f"hidden_stride={hidden_states.stride()} " - f"hidden_dtype={hidden_states.dtype} " - f"hidden_contiguous={hidden_states.is_contiguous()} " - f"residual={None if residual is None else tuple(residual.shape)} " - f"native={use_native_final_norm}", - flush=True, - ) - torch.cuda.synchronize() - print( - "SGLANG_TRACE_QWEN35_FINAL_NORM stage=pre_sync_returned", - flush=True, - ) if residual is None: - hidden_states = ( - self.norm.forward_native(hidden_states) - if use_native_final_norm - else self.norm(hidden_states) - ) + hidden_states = self.norm(hidden_states) else: - hidden_states, _ = ( - self.norm.forward_native(hidden_states, residual) - if use_native_final_norm - else self.norm(hidden_states, residual) - ) - if trace_final_norm: - print( - "SGLANG_TRACE_QWEN35_FINAL_NORM stage=post_sync_enter", - flush=True, - ) - torch.cuda.synchronize() - print( - "SGLANG_TRACE_QWEN35_FINAL_NORM stage=post_sync_returned", - flush=True, - ) + hidden_states, _ = self.norm(hidden_states, residual) if len(aux_hidden_states) == 0: return hidden_states diff --git a/python/sglang/srt/runtime_context.py b/python/sglang/srt/runtime_context.py index ed8ac84d61..5a6e82b217 100644 --- a/python/sglang/srt/runtime_context.py +++ b/python/sglang/srt/runtime_context.py @@ -660,9 +660,11 @@ class ForwardFlags: # fuse_mlp_allreduce: next residual+LN absorbs the post-MLP all-reduce. # mlp_reduce_scatter: postprocess will reduce-scatter (skip MLP AR). # flashinfer_trtllm_bypass: deepseek dual-stream graph topk bypass. + # defer_moe_finalize: next layer's fused collective absorbs the finalize. "fuse_mlp_allreduce": False, "mlp_reduce_scatter": False, "flashinfer_trtllm_bypass": False, + "defer_moe_finalize": False, # LayerNorm sequence parallelism region; see layers/layernorm_sp.py. "sp_active": False, } @@ -679,6 +681,7 @@ class ForwardFlags: "fuse_mlp_allreduce", "mlp_reduce_scatter", "flashinfer_trtllm_bypass", + "defer_moe_finalize", "sp_active", } ) diff --git a/test/registered/e2e/models/test_dsa_glm52_nvfp4_tp_mtp.py b/test/registered/e2e/models/test_dsa_glm52_nvfp4_tp_mtp.py index a7d53b3283..7e0b3a76df 100644 --- a/test/registered/e2e/models/test_dsa_glm52_nvfp4_tp_mtp.py +++ b/test/registered/e2e/models/test_dsa_glm52_nvfp4_tp_mtp.py @@ -28,6 +28,8 @@ class TestGLM52NVFP4TPMTP( "flashinfer_trtllm", "--quantization", "modelopt_fp4", + "--flashinfer-allreduce-fusion-backend", + "cutedsl", ] diff --git a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py new file mode 100644 index 0000000000..35e5fe04da --- /dev/null +++ b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py @@ -0,0 +1,331 @@ +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch + +from sglang.srt.layers.communicator import LayerCommunicator +from sglang.srt.layers.flashinfer_mnnvl_cutedsl import ( + FlashInferMNNVLCuteDSLARFusion, + _retargeted_config, + _with_early_finalize_shared_load, +) +from sglang.srt.layers.layernorm import RMSNorm +from sglang.srt.layers.moe.cutedsl_ar_fusion import ( + CuteDSLFusionLayerCommunicator, + MoeFinalizeHandoff, + install_cutedsl_fusion, + prepare_cutedsl_fusion, +) +from sglang.srt.model_executor.cuda_graph_config import CudaGraphConfig, PhaseConfig +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.runtime_context import get_forward, publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=9, suite="base-a-test-cpu") + +_MODULE = "sglang.srt.layers.moe.cutedsl_ar_fusion" +_DECODE = SimpleNamespace(forward_mode=ForwardMode.DECODE, input_ids=torch.zeros(8)) + + +def _communicator(): + comm = CuteDSLFusionLayerCommunicator.__new__(CuteDSLFusionLayerCommunicator) + comm.input_layernorm = RMSNorm(8, eps=1e-6) + comm.post_attention_layernorm = RMSNorm(8, eps=1e-6) + return comm + + +def _install(layers, **kwargs): + reset_context() + publish(ServerArgs(model_path="dummy"), role="test") + return install_cutedsl_fusion( + layers, + hidden_size=8, + top_k=2, + rms_epsilon=1e-6, + can_defer_finalize=lambda layer: False, + label="test", + **kwargs, + ) + + +@pytest.fixture +def eligible(): + """Every shared gate open, so a case varies only the predicate it names.""" + with ( + patch.object( + CuteDSLFusionLayerCommunicator, "_common_eligible", return_value=True + ), + patch( + f"{_MODULE}.get_exec", + return_value=SimpleNamespace( + comm=SimpleNamespace(enable_quant_communications=False) + ), + ), + ): + yield + + +def test_last_layer_consumes_but_does_not_skip_the_pending_all_reduce(eligible): + """The last layer must fuse the reduction its predecessor skipped, or the + tagged tensor reaches the final norm unreduced; it must not skip its own.""" + last = _communicator() + last.successor_absorbs_all_reduce = False + last.fusion_service = SimpleNamespace( + all_reduce_residual_rms_norm=lambda *, local_contribution, residual, gamma: ( + local_contribution + 1, + residual + 1, + ) + ) + hidden_states = torch.zeros(8, 8) + hidden_states._sglang_needs_allreduce_fusion = True + + with ( + patch.object( + CuteDSLFusionLayerCommunicator, + "_finish_prepare_attn", + lambda self, *, hidden_states, residual, forward_batch: ( + hidden_states, + residual, + ), + ), + patch.object( + LayerCommunicator, + "prepare_attn", + lambda *a, **k: pytest.fail("fell through to the unfused path"), + ), + ): + out_hidden, _ = last.prepare_attn(hidden_states, torch.zeros(8, 8), _DECODE) + + assert torch.equal(out_hidden, torch.ones(8, 8)) + assert last._can_absorb_post_moe_all_reduce(_DECODE, 8) is False + + +def test_a_replicated_output_producer_keeps_its_own_all_reduce(eligible): + """A TP1 shared expert (or TP1 dense MLP) is not partial; handing its layer's + reduction onward would scale the replicated output by tp_size.""" + layers = [ + SimpleNamespace(layer_communicator=_communicator(), replicated=replicated) + for replicated in (True, False, False) + ] + _install(layers, requires_local_reduction=lambda layer: layer.replicated) + replicated, plain = (layer.layer_communicator for layer in layers[:2]) + + with patch.object( + LayerCommunicator, + "should_fuse_mlp_allreduce_with_next_layer", + return_value=False, + ): + assert replicated.should_fuse_mlp_allreduce_with_next_layer(_DECODE) is False + assert plain.should_fuse_mlp_allreduce_with_next_layer(_DECODE) is True + # Consuming what a predecessor skipped stays independently eligible. + assert replicated._can_consume_post_moe_all_reduce(_DECODE, 8) is True + + +def test_a_service_nested_under_a_wrapper_is_prepared(): + """A VLM wrapper holds the model as a submodule and has no pre-capture hook; + an unprepared service declines every M, leaving no fusion at all.""" + prepared = [] + layer = torch.nn.Linear(2, 2) + layer.layer_communicator = _communicator() + wrapper = torch.nn.Module() + wrapper.language_model = torch.nn.Sequential(layer) + _install([layer]) + layer.layer_communicator.fusion_service.prepare = lambda *, max_m: prepared.append( + max_m + ) + + # The workspace M bound is the largest of every framework source. + reset_context() + publish( + ServerArgs( + model_path="dummy", + cuda_graph_config=CudaGraphConfig( + decode=PhaseConfig(max_bs=512, bs=[1, 64, 256]), + prefill=PhaseConfig(max_bs=4096, bs=[1024, 2048, 4096]), + ), + ), + role="test", + ) + with patch(f"{_MODULE}.cutedsl_moe_max_num_tokens", return_value=8192): + prepare_cutedsl_fusion(wrapper, max_running_requests=2048) + assert prepared == [8192] + + with pytest.raises(ValueError, match="installed no CuTe DSL fusion service"): + prepare_cutedsl_fusion(torch.nn.Linear(2, 2), max_running_requests=2048) + + +def test_wrapper_passes_no_routed_scaling_factor_to_the_kernel(): + """The routed output is already scaled; forwarding the factor would scale + DeepSeek's routed contribution twice.""" + calls = [] + wrapper = object.__new__(FlashInferMNNVLCuteDSLARFusion) + wrapper.hidden_size, wrapper.device = 8, torch.device("cpu") + wrapper.rms_epsilon, wrapper.weight_bias = 1e-6, 0.0 + wrapper.workspace = object() + wrapper.supports = lambda m: True + wrapper._patterns = SimpleNamespace( + kARResidualRMSNorm=1, kMoEFinalizeARResidualRMSNorm=7 + ) + wrapper._allreduce_fusion = lambda **kwargs: calls.append(kwargs) + x = torch.empty(4, 8, dtype=torch.bfloat16) + wrapper.moe_finalize_all_reduce_rms_norm( + routed_output=torch.empty(8, 8, dtype=torch.bfloat16), + expert_weights=torch.empty(4, 2, dtype=torch.bfloat16), + permuted_indices=torch.empty(4, 2, dtype=torch.int32), + gated_shared_output=x, + residual=x, + gamma=torch.empty(8, dtype=torch.bfloat16), + ) + wrapper.all_reduce_residual_rms_norm( + local_contribution=x, residual=x, gamma=torch.empty(8, dtype=torch.bfloat16) + ) + + assert [call["pattern"] for call in calls] == [7, 1] + assert all("routed_scaling_factor" not in call for call in calls) + + +def test_the_handoff_views_the_producer_storage(): + """The handoff is consumed in place by the next layer's kernel; a copy would + cost the [M*top_k, hidden] round trip the fusion exists to save.""" + m, top_k = 3, 10 + gemm2_out = torch.empty(m * top_k + 4, 16, dtype=torch.bfloat16) + expert_weights = torch.empty(m + 1, top_k, dtype=torch.bfloat16) + permuted_indices = torch.empty(m + 1, top_k, dtype=torch.int32) + handoff = MoeFinalizeHandoff.from_flashinfer( + SimpleNamespace( + gemm2_out=gemm2_out, + expert_weights=expert_weights, + expanded_idx_to_permuted_idx=permuted_indices, + top_k=top_k, + ), + gated_shared_output=torch.empty(m, 16, dtype=torch.bfloat16), + m=m, + ) + + assert handoff.routed_output.data_ptr() == gemm2_out.data_ptr() + assert handoff.permuted_indices.data_ptr() == permuted_indices.data_ptr() + assert tuple(handoff.expert_weights.shape) == (m, top_k) + + +def test_early_shared_load_touches_only_the_finalize_routes(): + """Only a fused finalize has a completed shared-expert handoff to load early; + the standalone all-reduce must keep the safe ordering.""" + from flashinfer.comm.mnnvl_cutedsl import DEFAULT_CONFIG + from flashinfer.comm.mnnvl_cutedsl.kernel_ht import HTFinalizeTuning + + config = _with_early_finalize_shared_load(DEFAULT_CONFIG) + + for before, after in zip(DEFAULT_CONFIG.profiles, config.profiles): + assert after.all_reduce_routes == before.all_reduce_routes + for target in after.finalize_routes.targets: + # HT has no shared-load ordering option. + if not isinstance(target.preset, HTFinalizeTuning): + assert target.preset.load_shared_expert_before_pdl is True + + +def test_dual_stream_op_pins_the_deferral_off_under_a_deferring_caller(): + """The op's Tensor schema cannot carry a handoff; the dispatcher would raise + "Unable to cast ... to Tensor".""" + from sglang.srt.models.deepseek_v2 import ( # noqa: F401 (registers the op) + dsv2_flashinfer_moe_dual_stream_graph, + ) + + class _DeferRecordingMoE: + def forward_normal_dual_stream(self, hidden_states): + self.seen_defer = get_forward().defer_moe_finalize + return object() if self.seen_defer else hidden_states + 1 + + reset_context() + fusion = _DeferRecordingMoE() + op = torch.ops.sglang.dsv2_flashinfer_moe_dual_stream_graph.default + # The CUDA key runs the real schema while the stub keeps tensors on CPU. + cuda_key = torch._C.DispatchKeySet(torch._C.DispatchKey.CUDA) + with ( + get_forward().scoped(defer_moe_finalize=True), + patch( + "sglang.srt.models.deepseek_v2.get_tc_piecewise_forward_context", + return_value=SimpleNamespace(moe_fusions={0: fusion}), + ), + ): + out = op.redispatch(cuda_key, torch.zeros(4, 8), 0, True, False) + assert get_forward().defer_moe_finalize is True + + assert fusion.seen_defer is False + assert torch.equal(out, torch.ones(4, 8)) + + +def _flashinfer_accepts(preset, *, hidden_size, top_k, tp_size): + """FlashInfer's own HT kernel validation, which runs before any compile.""" + from flashinfer.comm.mnnvl_cutedsl.kernel_ht.device_kernel import ( + _MoeFinalizeAllReduceRMSNormHTDeviceKernel, + ) + + _MoeFinalizeAllReduceRMSNormHTDeviceKernel( + hidden=hidden_size, + top_k=top_k, + tp=tp_size, + rank=0, + # Arbitrary grid when the preset leaves it to the device's SM count. + active_ctas=preset.persistent_ctas or tp_size * 8, + stages=preset.stages, + consumer_threads=preset.consumer_threads, + vectors_per_thread=preset.vectors_per_thread, + reduction_warps=preset.reduction_warps, + reduction_cta_groups=preset.reduction_cta_groups, + rms_token_groups=preset.rms_token_groups, + rms_pipeline_stages=preset.rms_pipeline_stages, + rms_shard_major=preset.rms_shard_major, + rms_epsilon=1e-6, + routed_scaling_factor=1.0, + weight_bias=0.0, + include_shared_expert=True, + add_residual=True, + write_residual_output=True, + enable_pdl=preset.enable_pdl, + ) + + +# (hidden_size, top_k, tp_size, HT routable), from the checkpoint configs of +# Qwen3.8, DeepSeek-V3 and GLM-5.3. False: the vectors per reduction shard are +# not a warp multiple at that width, which the kernel rejects. +@pytest.mark.parametrize( + ("hidden_size", "top_k", "tp_size", "ht_routable"), + [ + (8192, 10, 8, True), + (8192, 10, 16, True), + (7168, 8, 4, True), + (7168, 8, 8, False), + (7168, 8, 16, False), + (6144, 8, 4, True), + (6144, 8, 8, True), + (6144, 8, 16, False), + ], +) +def test_retargeted_profiles_cover_capacity_and_pass_flashinfer_validation( + hidden_size, top_k, tp_size, ht_routable +): + """A wrong split aborts at compile time on a Blackwell node, and an + unroutable HT must fall back to LL and BT rather than lose the profile.""" + from flashinfer.comm.mnnvl_cutedsl import ProtocolKind + + profile = _retargeted_config(tp_size, hidden_size, top_k).profiles[0] + profile.validate_capacity(4096) + for routes, routed_top_k in ( + (profile.finalize_routes, top_k), + (profile.all_reduce_routes, 0), + ): + assert routes.is_unbounded + ht = [t.preset for t in routes.targets if t.protocol is ProtocolKind.HT] + assert bool(ht) is ht_routable + for preset in ht: + _flashinfer_accepts( + preset, hidden_size=hidden_size, top_k=routed_top_k, tp_size=tp_size + ) + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/unit/layers/moe/test_qwen35_flashinfer_fusion.py b/test/registered/unit/layers/moe/test_qwen35_flashinfer_fusion.py deleted file mode 100644 index 6288c41bf4..0000000000 --- a/test/registered/unit/layers/moe/test_qwen35_flashinfer_fusion.py +++ /dev/null @@ -1,220 +0,0 @@ -from dataclasses import dataclass -from types import SimpleNamespace -from unittest.mock import patch - -import pytest -import torch - -from sglang.srt.layers.flashinfer_mnnvl_cutedsl import ( - FlashInferMNNVLCuteDSLARFusion, - _with_early_finalize_shared_load, -) -from sglang.srt.layers.moe.qwen35_flashinfer_fusion import ( - Qwen35MoeFinalizeHandoff, - is_supported_forward_mode, - resolve_max_m, -) -from sglang.srt.model_executor.cuda_graph_config import ( - CudaGraphConfig, - PhaseConfig, -) -from sglang.srt.model_executor.forward_batch_info import ForwardMode -from sglang.srt.models.qwen3_5_text import Qwen3_5ForCausalLM -from sglang.srt.runtime_context import publish, reset_context -from sglang.srt.server_args import ServerArgs -from sglang.test.ci.ci_register import register_cpu_ci - -register_cpu_ci(est_time=9, suite="base-a-test-cpu") - - -@dataclass(frozen=True) -class _TestPreset: - load_shared_expert_before_pdl: bool = False - - -@dataclass(frozen=True) -class _TestTarget: - preset: object - - -@dataclass(frozen=True) -class _TestRoutes: - targets: tuple[_TestTarget, ...] - - -@dataclass(frozen=True) -class _TestProfile: - finalize_routes: _TestRoutes - - -@dataclass(frozen=True) -class _TestConfig: - profiles: tuple[_TestProfile, ...] - - -@pytest.mark.parametrize( - ("forward_mode", "expected"), - [ - (ForwardMode.DECODE, True), - (ForwardMode.EXTEND, True), - (ForwardMode.IDLE, False), - (ForwardMode.TARGET_VERIFY, True), - (ForwardMode.DRAFT_EXTEND_V2, False), - ], -) -def test_supported_forward_modes(forward_mode, expected): - assert is_supported_forward_mode(forward_mode) is expected - - -@patch( - "sglang.srt.layers.moe.qwen35_flashinfer_fusion.cutedsl_moe_max_num_tokens", - return_value=8192, -) -def test_framework_capacity_is_maximum_of_all_sources(_cutedsl_moe_max_num_tokens): - # The graph bounds are a bag leaf, so the test states them by publishing. - reset_context() - publish( - ServerArgs( - model_path="dummy", - cuda_graph_config=CudaGraphConfig( - decode=PhaseConfig(max_bs=512, bs=[1, 64, 256]), - prefill=PhaseConfig(max_bs=4096, bs=[1024, 2048, 4096]), - ), - ), - role="test", - ) - runner = SimpleNamespace(server_args=SimpleNamespace(), max_running_requests=2048) - - assert resolve_max_m(runner) == 8192 - - -def test_deferred_handoff_reuses_producer_storage(): - m, top_k, hidden_size = 3, 10, 16 - gemm2_out = torch.empty(m * top_k + 4, hidden_size, dtype=torch.bfloat16) - expert_weights = torch.empty(m, top_k, dtype=torch.bfloat16) - permuted_indices = torch.empty(m, top_k, dtype=torch.int32) - gated_shared_output = torch.empty(m, hidden_size, dtype=torch.bfloat16) - deferred = SimpleNamespace( - gemm2_out=gemm2_out, - expert_weights=expert_weights, - expanded_idx_to_permuted_idx=permuted_indices, - top_k=top_k, - ) - - handoff = Qwen35MoeFinalizeHandoff.from_flashinfer( - deferred, - gated_shared_output=gated_shared_output, - m=m, - ) - - assert handoff.routed_output.data_ptr() == gemm2_out.data_ptr() - assert handoff.expert_weights.data_ptr() == expert_weights.data_ptr() - assert handoff.permuted_indices.data_ptr() == permuted_indices.data_ptr() - assert handoff.gated_shared_output is gated_shared_output - - -def test_qwen_workspace_config_enables_only_supported_finalize_presets(): - untouched_preset = object() - default_config = _TestConfig( - profiles=( - _TestProfile( - finalize_routes=_TestRoutes( - targets=( - _TestTarget(_TestPreset()), - _TestTarget(untouched_preset), - ) - ) - ), - ) - ) - - qwen_config = _with_early_finalize_shared_load(default_config) - - assert qwen_config is not default_config - assert ( - default_config.profiles[0] - .finalize_routes.targets[0] - .preset.load_shared_expert_before_pdl - is False - ) - assert ( - qwen_config.profiles[0] - .finalize_routes.targets[0] - .preset.load_shared_expert_before_pdl - is True - ) - assert qwen_config.profiles[0].finalize_routes.targets[1].preset is untouched_preset - - -def test_wrapper_calls_only_the_stable_unified_api(): - calls = [] - wrapper = object.__new__(FlashInferMNNVLCuteDSLARFusion) - wrapper.hidden_size = 8 - wrapper.top_k = 2 - wrapper.max_m = 4 - wrapper.rms_epsilon = 1e-5 - wrapper.weight_bias = 0.0 - wrapper.device = torch.device("cpu") - wrapper.workspace = object() - wrapper.supports = lambda m: True - wrapper._patterns = SimpleNamespace( - kARResidualRMSNorm=1, - kMoEFinalizeARResidualRMSNorm=7, - ) - wrapper._allreduce_fusion = lambda **kwargs: calls.append(kwargs) - - routed_output = torch.empty(8, 8, dtype=torch.bfloat16) - expert_weights = torch.empty(4, 2, dtype=torch.bfloat16) - permuted_indices = torch.empty(4, 2, dtype=torch.int32) - gated_shared_output = torch.empty(4, 8, dtype=torch.bfloat16) - residual = torch.empty(4, 8, dtype=torch.bfloat16) - gamma = torch.empty(8, dtype=torch.bfloat16) - norm_output = torch.empty_like(residual) - residual_output = torch.empty_like(residual) - - wrapper.moe_finalize_all_reduce_rms_norm( - routed_output=routed_output, - expert_weights=expert_weights, - permuted_indices=permuted_indices, - gated_shared_output=gated_shared_output, - residual=residual, - gamma=gamma, - norm_output=norm_output, - residual_output=residual_output, - ) - - assert calls[0]["launch_with_pdl"] is True - assert "routed_scaling_factor" not in calls[0] - - wrapper.all_reduce_residual_rms_norm( - local_contribution=residual, - residual=residual, - gamma=gamma, - norm_output=norm_output, - residual_output=residual_output, - ) - - assert calls[1]["pattern"] == 1 - assert calls[1]["launch_with_pdl"] is True - assert "routed_scaling_factor" not in calls[1] - assert "expanded_idx_to_permuted_idx" not in calls[1] - - -def test_text_entry_wrapper_delegates_pre_capture_prepare(): - calls = [] - runner = object() - wrapper = SimpleNamespace( - model=SimpleNamespace( - prepare_before_cuda_graph_capture=lambda value: calls.append(value) - ) - ) - - Qwen3_5ForCausalLM.prepare_before_cuda_graph_capture(wrapper, runner) - - assert calls == [runner] - - -if __name__ == "__main__": - import sys - - sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py index e91a47dc07..c27a4a9638 100644 --- a/test/registered/unit/layers/test_communicator_ffn_exit.py +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -105,6 +105,32 @@ class TestFfnExit(CustomTestCase): self.assertEqual(seen, (False, False)) communicator.postprocess_layer.assert_called_once() + def test_deferral_implies_fusion_and_passes_the_handoff_through(self): + """A deferring communicator publishes both flags, and a non-tensor + handoff leaves finish() untouched for the next layer's input norm.""" + + class Defers(LayerCommunicator): + def should_defer_moe_finalize(self, forward_batch, m=None): + return True + + def should_fuse_mlp_allreduce_with_next_layer(self, forward_batch): + return False + + communicator = make_communicator(fuse=False, reduce_scatter=False, cls=Defers) + handoff = object() + with communicator.ffn_exit(self.forward_batch) as ffn_exit: + seen = published_flags() + (get_forward().defer_moe_finalize,) + self.assertEqual(seen, (True, False, True)) + self.assertEqual( + ffn_exit.finish(handoff, self.residual), (handoff, self.residual) + ) + communicator.postprocess_layer.assert_not_called() + self.assertFalse(get_forward().defer_moe_finalize) + + # The MoE may decline per forward; a tensor then takes the fused exit. + hidden_states, _ = ffn_exit.finish(self.hidden_states * 2, self.residual) + self.assertTrue(hidden_states._sglang_needs_allreduce_fusion) + def test_compiles_without_graph_breaks(self): communicator = LayerCommunicator.__new__(LayerCommunicator) communicator.should_fuse_mlp_allreduce_with_next_layer = lambda forward_batch: (