Refactor the Cute-DSL AR fusion to support DeepseekV2 archs (GLM-5.3, etc.) (#39816)

Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
Co-authored-by: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com>
This commit is contained in:
Brayden Zhong
2026-09-24 06:00:58 -07:00
committed by GitHub
co-authored by Mohammad Angkad Mohammad Miadh Angkad
parent ce06a14444
commit 3177d10ca6
20 changed files with 1327 additions and 855 deletions
+4 -1
View File
@@ -630,7 +630,7 @@ class ExecComm(msgspec.Struct):
"Enforce disable FlashInfer allreduce fusion.",
] = False
flashinfer_allreduce_fusion_backend: A[
Optional[Literal["auto", "trtllm", "mnnvl"]],
Optional[Literal["auto", "trtllm", "mnnvl", "cutedsl"]],
Arg(
help=(
"Enable FlashInfer allreduce fusion and choose backend. "
@@ -641,6 +641,9 @@ class ExecComm(msgspec.Struct):
"'trtllm': available on single-node systems only. "
"'mnnvl': available on SM90 single-node systems and SM100/SM103 "
"single-node or multi-node systems via MNNVL fabric. "
"'cutedsl': Blackwell-only bf16 MNNVL CuTe DSL backend; also "
"fuses the MoE finalize and the shared-expert add into the "
"collective when the MoE runner can defer them. "
"Fuses allreduce with Residual + RMSNorm for supported MoE models."
),
resolvable=True,
+1
View File
@@ -202,6 +202,7 @@ def validate_flashinfer_megamoe_model(server_args: Any) -> None:
"DeepseekV32ForCausalLM",
"DeepseekV4ForCausalLM",
"Glm4MoeForCausalLM",
"GlmMoeDsaForCausalLM",
"NemotronHForCausalLM",
"NemotronHPuzzleForCausalLM",
"Qwen2MoeForCausalLM",
-14
View File
@@ -920,20 +920,6 @@ def _flashinfer_allreduce_fusion_auto_enable(view: Any) -> dict:
and view.nnodes == 1
and not view.disable_custom_all_reduce
)
if envs.SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION.get() and model_arch in {
"Qwen3_5MoeForCausalLM",
"Qwen3_5MoeForConditionalGeneration",
}:
# The Qwen backend owns one workspace for ordinary AR and MoE finalize;
# do not allocate or fall back to the legacy TRTLLM/MNNVL workspace.
if view.flashinfer_allreduce_fusion_backend is not None:
logger.warning(
"SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION owns both Qwen3.5 "
"AllReduce fusion patterns; suppressing the separately configured "
"--flashinfer-allreduce-fusion-backend=%s",
view.flashinfer_allreduce_fusion_backend,
)
return {"flashinfer_allreduce_fusion_backend": None}
if (
view.flashinfer_allreduce_fusion_backend is None
and model_arch in _FLASHINFER_ALLREDUCE_FUSION_ARCHS
@@ -236,8 +236,12 @@ def _set_all_reduce_flags() -> None:
set_custom_all_reduce(not get_exec().comm.disable_custom_all_reduce)
set_mscclpp_all_reduce(get_exec().comm.enable_mscclpp)
set_torch_symm_mem_all_reduce(get_exec().comm.enable_torch_symm_mem)
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
set_flashinfer_allreduce_only(
get_exec().comm.flashinfer_allreduce_fusion_backend is not None
# cutedsl has no legacy workspace for tagged groups to reduce over.
and not uses_cutedsl_ar_fusion()
)
+15 -7
View File
@@ -1244,6 +1244,9 @@ class Envs:
SGLANG_NIXL_EP_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(128)
SGLANG_PPLX_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(128)
SGLANG_ENABLE_MOE_DEFERRED_FINALIZE = EnvBool(True)
# The [M*top_k, hidden] HBM round trip pays only at small M; 0 disables.
# GLM-5.2 (H=6144): neutral at 192 on B300 TP8, crossover 192-223 on GB300 TP4.
SGLANG_MOE_DEFERRED_FINALIZE_MAX_TOKENS = EnvInt(192)
# DeepSeek/GLM MoE (deepseek_v2.py): quantize the (dp-gathered) MoE input
# to per-token-group-128 fp8 ONCE and feed both the fused shared-expert
# GEMM (cutlass w8a8 linear) and the routed experts' triton fused runner,
@@ -1769,13 +1772,6 @@ class Envs:
# Qwen3.5 and GDN
SGLANG_ENABLE_GDN_DECODE_FUSED_PROJ_CONV = EnvBool(True)
SGLANG_TRACE_QWEN35_FINAL_NORM = EnvBool(False)
SGLANG_QWEN35_NATIVE_FINAL_NORM = EnvBool(False)
# One switch enables deferred MoE finalize and AR + residual + RMSNorm.
SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION = EnvBool(False)
# Distinct workspace configurations allowed in one process. Production
# uses one model/configuration per rank, so fail closed on accidental reuse.
SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION_MAX_INSTANCES = EnvInt(1)
# ===================================================================
# Plugin system
@@ -1887,10 +1883,22 @@ class _DeprecatedEnv:
# ad-hoc warnings. For a rename where the old name must keep working through a
# descriptor, use EnvBoolWithAlias / EnvIntWithAlias instead.
_DEPRECATED_ENVS: Dict[str, _DeprecatedEnv] = {
"SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION": _DeprecatedEnv(
note=(
"Pass --flashinfer-allreduce-fusion-backend cutedsl instead. "
"Without it an eligible model auto-enables the legacy mnnvl "
"backend rather than the CuTe DSL fusion."
)
),
"SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION_MAX_INSTANCES": _DeprecatedEnv(
note="One workspace per process is now an invariant, not a limit."
),
# Removed without replacement.
"SGLANG_ENABLE_CP_V2": _DeprecatedEnv(
note="Strategy-based prefill context parallelism is now the only generic implementation."
),
"SGLANG_TRACE_QWEN35_FINAL_NORM": _DeprecatedEnv(),
"SGLANG_QWEN35_NATIVE_FINAL_NORM": _DeprecatedEnv(),
"SGLANG_ENABLE_HICACHE_BUFFER_ANCHOR_LOCK": _DeprecatedEnv(
note="Buffer-mode anchor pinning is always on; set "
"SGLANG_HICACHE_BUFFER_ANCHOR_LOCK_CAP=0 to disable it."
+32 -3
View File
@@ -58,7 +58,10 @@ from sglang.srt.layers.dp_attention import (
is_enable_moe_cp_allgather,
moe_cp_all_gather_into_tensor,
)
from sglang.srt.layers.flashinfer_comm_fusion import is_flashinfer_allreduce_unavailable
from sglang.srt.layers.flashinfer_comm_fusion import (
is_flashinfer_allreduce_unavailable,
uses_cutedsl_ar_fusion,
)
from sglang.srt.layers.moe import (
can_merge_post_experts_all_reduce,
deferred_post_experts_all_reduce,
@@ -198,6 +201,8 @@ def apply_flashinfer_allreduce_fusion(batch_size: int):
and _is_flashinfer_available
and not is_dp_attention_enabled()
and get_exec().comm.flashinfer_allreduce_fusion_backend is not None
# cutedsl runs its own fused path from the fusion communicator.
and not uses_cutedsl_ar_fusion()
and not is_flashinfer_allreduce_unavailable()
# Symbolic size checks stay last: under Dynamo tracing they guard on
# the dynamic token dim, so statically-off configs must short-circuit
@@ -859,6 +864,14 @@ class LayerCommunicator:
post_residual_addition,
)
return self._finish_prepare_attn(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
)
def _finish_prepare_attn(self, hidden_states, residual, forward_batch):
"""Tail every prepare_attn path must run, or ``attn_inputs`` is unset."""
hidden_states = self._communicate_simple_fn(
hidden_states=hidden_states,
forward_batch=forward_batch,
@@ -953,6 +966,12 @@ class LayerCommunicator:
return True
return False
def should_defer_moe_finalize(
self, forward_batch: ForwardBatch, m: int | None = None
) -> bool:
"""Whether the MoE may hand an unfinalized output to the next layer."""
return False
# NOTE: This function will cause torch recompilation
def should_fuse_mlp_allreduce_with_next_layer(
self, forward_batch: ForwardBatch
@@ -1046,11 +1065,13 @@ def scatter_mode_layouts(
class FfnExit:
"""One FFN's reduction decision. Inside the ``with`` block it is published as
``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` on ``get_forward()``."""
``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` / ``defer_moe_finalize`` on
``get_forward()``."""
__slots__ = (
"communicator",
"forward_batch",
"defer_moe_finalize",
"fuse_mlp_allreduce",
"mlp_reduce_scatter",
"_scope",
@@ -1059,13 +1080,17 @@ class FfnExit:
def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch):
self.communicator = communicator
self.forward_batch = forward_batch
self.defer_moe_finalize = communicator.should_defer_moe_finalize(forward_batch)
# Deferring implies fusing: a handoff skips the post-experts all-reduce.
self.fuse_mlp_allreduce = (
communicator.should_fuse_mlp_allreduce_with_next_layer(forward_batch)
self.defer_moe_finalize
or communicator.should_fuse_mlp_allreduce_with_next_layer(forward_batch)
)
self.mlp_reduce_scatter = communicator.should_use_reduce_scatter(forward_batch)
self._scope = get_forward().scoped(
fuse_mlp_allreduce=self.fuse_mlp_allreduce,
mlp_reduce_scatter=self.mlp_reduce_scatter,
defer_moe_finalize=self.defer_moe_finalize,
)
def __enter__(self) -> "FfnExit":
@@ -1079,6 +1104,10 @@ class FfnExit:
self, hidden_states: torch.Tensor, residual: torch.Tensor
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Leave the reduction to the next layer's input norm, or postprocess."""
if not isinstance(hidden_states, torch.Tensor):
# A deferred MoE finalize handoff, consumed by the next prepare_attn.
assert self.defer_moe_finalize, "unrequested deferred MoE handoff"
return hidden_states, residual
if self.fuse_mlp_allreduce:
hidden_states._sglang_needs_allreduce_fusion = True
return hidden_states, residual
@@ -47,6 +47,14 @@ def _resolve_backend(backend: str, is_multi_node: bool = False) -> str:
"FlashInfer allreduce fusion requires SM90 or SM10X NVIDIA GPUs."
)
if backend == "cutedsl":
if not get_platform().is_sm100:
raise ValueError(
"FlashInfer allreduce fusion cutedsl backend requires a "
"Blackwell system."
)
return backend
if backend == "auto":
if is_multi_node:
if get_platform().is_sm100:
@@ -72,6 +80,11 @@ def _resolve_backend(backend: str, is_multi_node: bool = False) -> str:
return backend
def uses_cutedsl_ar_fusion() -> bool:
"""Selected CuTe DSL owns both patterns, so the legacy workspace stands down."""
return get_exec().comm.flashinfer_allreduce_fusion_backend == "cutedsl"
def resolve_flashinfer_allreduce_fusion_backend() -> Optional[str]:
"""The fusion backend for this process, or None when fusion is off.
@@ -713,7 +726,7 @@ def ensure_workspace_initialized(
use_attn_tp_group: bool = True,
):
"""Ensure workspace is initialized."""
if _flashinfer_allreduce_unavailable:
if _flashinfer_allreduce_unavailable or uses_cutedsl_ar_fusion():
return False
if not is_flashinfer_available() or _flashinfer_comm is None:
@@ -3,9 +3,7 @@
from __future__ import annotations
import logging
import threading
from dataclasses import dataclass, replace
from functools import lru_cache
from dataclasses import replace
from typing import TYPE_CHECKING
import torch
@@ -20,8 +18,7 @@ logger = logging.getLogger(__name__)
def _import_kernel_backend():
# Imported here rather than at module scope: the CuTe DSL backend drags in
# CUDA-only dependencies that CPU-side importers of this module never need.
# Deferred: the backend drags in CUDA-only deps CPU importers never need.
from flashinfer.comm import AllReduceFusionPattern, allreduce_fusion
from flashinfer.comm.mnnvl_cutedsl import DEFAULT_CONFIG
from flashinfer.comm.mnnvl_cutedsl_ar import MNNVLCuteDSLAllReduceFusionWorkspace
@@ -34,6 +31,252 @@ def _import_kernel_backend():
)
# Mirrors the FlashInfer HT device kernel: warp size, bf16 elements per 16B vector.
_WARP_SIZE = 32
_VEC_BF16 = 8
# Kernel bound, with two non-consumer warps beside the reduction warps:
# consumer_threads + (2 + reduction_warps) * WARP_SIZE <= _CUDA_BLOCK_THREADS.
_CUDA_BLOCK_THREADS = 1024
_HT_NON_REDUCTION_WARPS = 2
_HT_REDUCTION_WARP_CHOICES = (1, 2, 4, 8)
_SUPPORTED_TP_SIZES = (2, 4, 8, 16)
def _ht_shard_split(
hidden_size: int, max_consumer_threads: int
) -> tuple[int, int] | None:
# The kernel shards a token into consumer_threads * 8 * vectors_per_thread
# elements, and consumer_threads must divide its 16-byte vector count.
packs = hidden_size // _VEC_BF16
# packs // 2 keeps vectors_per_thread >= 2, as in the GB300 presets.
limit = min(max_consumer_threads, packs // 2)
for consumer_threads in range(
limit - limit % _WARP_SIZE, _WARP_SIZE - 1, -_WARP_SIZE
):
if packs % consumer_threads == 0:
return consumer_threads, packs // consumer_threads
return None
def _ht_reduction_warp_order(preferred: int) -> tuple[int, ...]:
# Preset's value first, then nearest, fewer before more: each warp costs consumers.
return tuple(
sorted(
_HT_REDUCTION_WARP_CHOICES,
key=lambda warps: (warps > preferred, abs(warps - preferred)),
)
)
def _ht_shard_major_is_legal(
consumer_threads: int, rms_token_groups: int, tp_size: int
) -> bool:
# Shard-major RMS needs an integer number of reduction shards per RMS warp.
rms_warps_per_token = (consumer_threads // rms_token_groups) // _WARP_SIZE
return (
rms_warps_per_token > 0
and tp_size >= rms_warps_per_token
and tp_size % rms_warps_per_token == 0
)
def _ht_retarget(preset, *, hidden_size: int, tp_size: int):
"""``preset`` re-aimed at this shape, or None when no legal split exists.
Only shape-dependent fields move, except ``rms_shard_major``, which the
kernel rejects unless tp is a multiple of the RMS warps per token.
"""
packs = hidden_size // _VEC_BF16
if packs % tp_size:
# Kernel: "hidden vector count must be divisible by tp".
return None
packs_per_shard = packs // tp_size
for reduction_warps in _ht_reduction_warp_order(preset.reduction_warps):
if packs_per_shard % (reduction_warps * _WARP_SIZE):
continue
split = _ht_shard_split(
hidden_size,
_CUDA_BLOCK_THREADS
- (_HT_NON_REDUCTION_WARPS + reduction_warps) * _WARP_SIZE,
)
if split is None:
continue
consumer_threads, vectors_per_thread = split
return replace(
preset,
consumer_threads=consumer_threads,
vectors_per_thread=vectors_per_thread,
reduction_warps=reduction_warps,
rms_shard_major=preset.rms_shard_major
and _ht_shard_major_is_legal(
consumer_threads, preset.rms_token_groups, tp_size
),
)
return None
def _routes(bounds, ll_target, bt_targets, ht_target):
# Without HT the widest BT range takes the unbounded slot, so the profile
# still covers the whole workspace capacity.
from flashinfer.comm.mnnvl_cutedsl import MRangeDispatch
if ht_target is not None:
return MRangeDispatch(
upper_bounds=bounds,
targets=(ll_target, *bt_targets, ht_target),
)
return MRangeDispatch(
upper_bounds=(*bounds[: len(bt_targets)], None),
targets=(ll_target, *bt_targets),
)
def _gb300_presets(*, wide_tp: bool):
"""The shipped GB300 (bounds, LL, BT pair, HT) per operation, TP16 or TP8."""
from flashinfer.comm.mnnvl_cutedsl.kernel_bt import (
BT_ALL_REDUCE_GB300_TP8_H8192_PRESET_0,
BT_ALL_REDUCE_GB300_TP8_H8192_PRESET_1,
BT_ALL_REDUCE_GB300_TP16_H8192_PRESET_0,
BT_ALL_REDUCE_GB300_TP16_H8192_PRESET_1,
BT_FINALIZE_GB300_TP8_H8192_K10_PRESET_0,
BT_FINALIZE_GB300_TP8_H8192_K10_PRESET_1,
BT_FINALIZE_GB300_TP16_H8192_K10_PRESET_0,
BT_FINALIZE_GB300_TP16_H8192_K10_PRESET_1,
)
from flashinfer.comm.mnnvl_cutedsl.kernel_ht import (
HT_ALL_REDUCE_GB300_TP8_H8192,
HT_ALL_REDUCE_GB300_TP16_H8192,
HT_FINALIZE_GB300_TP8_H8192_K10,
HT_FINALIZE_GB300_TP16_H8192_K10,
)
from flashinfer.comm.mnnvl_cutedsl.kernel_ll import (
LL_ALL_REDUCE_GB300_TP8_H8192,
LL_ALL_REDUCE_GB300_TP16_H8192,
LL_FINALIZE_GB300_TP8_H8192_K10,
LL_FINALIZE_GB300_TP16_H8192_K10,
)
# FlashInfer's measured GB300 crossovers.
if wide_tp:
return {
"finalize": (
(7, 52, 703, None),
LL_FINALIZE_GB300_TP16_H8192_K10,
(
BT_FINALIZE_GB300_TP16_H8192_K10_PRESET_0,
BT_FINALIZE_GB300_TP16_H8192_K10_PRESET_1,
),
HT_FINALIZE_GB300_TP16_H8192_K10,
),
"all_reduce": (
(5, 512, 959, None),
LL_ALL_REDUCE_GB300_TP16_H8192,
(
BT_ALL_REDUCE_GB300_TP16_H8192_PRESET_0,
BT_ALL_REDUCE_GB300_TP16_H8192_PRESET_1,
),
HT_ALL_REDUCE_GB300_TP16_H8192,
),
}
return {
"finalize": (
(23, 48, 703, None),
LL_FINALIZE_GB300_TP8_H8192_K10,
(
BT_FINALIZE_GB300_TP8_H8192_K10_PRESET_0,
BT_FINALIZE_GB300_TP8_H8192_K10_PRESET_1,
),
HT_FINALIZE_GB300_TP8_H8192_K10,
),
"all_reduce": (
(15, 256, 1024, None),
LL_ALL_REDUCE_GB300_TP8_H8192,
(
BT_ALL_REDUCE_GB300_TP8_H8192_PRESET_0,
BT_ALL_REDUCE_GB300_TP8_H8192_PRESET_1,
),
HT_ALL_REDUCE_GB300_TP8_H8192,
),
}
def _retargeted_config(tp_size: int, hidden_size: int, top_k: int):
"""A single-profile routing config for a shape FlashInfer does not ship.
Reuses the shipped GB300 crossovers, which were measured at tp=8/16
hidden=8192 top_k=10 and are unmeasured elsewhere.
"""
from flashinfer.comm.mnnvl_cutedsl import (
KernelTarget,
MNNVLCuteDSLConfig,
ProtocolKind,
StaticProfile,
)
shipped = _gb300_presets(wide_tp=tp_size >= 16)
ht = {
op: _ht_retarget(presets[3], hidden_size=hidden_size, tp_size=tp_size)
for op, presets in shipped.items()
}
if any(preset is None for preset in ht.values()):
# Both operations share the HT protocol state, so HT goes for both.
ht = dict.fromkeys(ht)
logger.warning(
"MNNVL CuTe DSL: hidden_size=%d admits no HT shard split at "
"tp_size=%d; serving this shape with the LL and BT routes only, "
"which costs throughput at large token counts.",
hidden_size,
tp_size,
)
def routes(op):
bounds, ll, bt, _ = shipped[op]
return _routes(
bounds,
KernelTarget(protocol=ProtocolKind.LL, preset=ll),
tuple(KernelTarget(protocol=ProtocolKind.BT, preset=p) for p in bt),
None
if ht[op] is None
else KernelTarget(protocol=ProtocolKind.HT, preset=ht[op]),
)
profile = StaticProfile(
tp_size=tp_size,
hidden_size=hidden_size,
top_k=top_k,
dtype=torch.bfloat16,
finalize_routes=routes("finalize"),
all_reduce_routes=routes("all_reduce"),
)
return MNNVLCuteDSLConfig(profiles=(profile,))
def _config_for_shape(default_config, *, tp_size: int, hidden_size: int, top_k: int):
# MNNVLCuteDSLConfig.resolve matches (tp, hidden, top_k, dtype) exactly and
# DEFAULT_CONFIG ships GB300 H=8192/K=10 only, so other shapes need one.
for profile in default_config.profiles:
if profile.matches(
tp_size=tp_size,
hidden_size=hidden_size,
top_k=top_k,
dtype=torch.bfloat16,
):
return default_config
logger.info(
"MNNVL CuTe DSL: no shipped profile for tp=%d hidden=%d top_k=%d; "
"re-targeting the GB300 presets at this shape. Their M crossovers were "
"measured at tp=8/16 hidden=8192 top_k=10, so benchmark against "
"--flashinfer-allreduce-fusion-backend mnnvl before deploying a shape "
"that matters.",
tp_size,
hidden_size,
top_k,
)
return _retargeted_config(tp_size, hidden_size, top_k)
def _with_early_finalize_shared_load(config):
profiles = []
updated_presets = 0
@@ -64,17 +307,6 @@ def _with_early_finalize_shared_load(config):
return replace(config, profiles=tuple(profiles))
@dataclass(frozen=True, slots=True)
class _WorkspaceSignature:
hidden_size: int
top_k: int
rms_epsilon: float
weight_bias: float
max_m: int
device_index: int
process_group_identity: int
class FlashInferMNNVLCuteDSLARFusion:
"""One graph-stable workspace serving both supported fusion patterns."""
@@ -100,8 +332,14 @@ class FlashInferMNNVLCuteDSLARFusion:
self.rms_epsilon = float(rms_epsilon)
self.weight_bias = float(weight_bias)
self.process_group = process_group
# Cached: supports() runs per layer and must not re-enter c10d.
self.tp_size = dist.get_world_size(process_group)
if self.tp_size not in _SUPPORTED_TP_SIZES:
raise ValueError(
f"MNNVL CuTe DSL fusion supports tp_size in {_SUPPORTED_TP_SIZES}, "
f"got {self.tp_size}"
)
self.device = torch.device(device)
self._destroyed = False
with torch.cuda.device(self.device):
self.device = torch.device("cuda", torch.cuda.current_device())
@@ -129,11 +367,14 @@ class FlashInferMNNVLCuteDSLARFusion:
self._patterns,
default_config,
) = _import_kernel_backend()
# Only fused finalize launches have a completed shared-expert handoff;
# standalone AllReduce kernels retain the safe load ordering.
shaped_config = _config_for_shape(
default_config,
tp_size=self.tp_size,
hidden_size=self.hidden_size,
top_k=self.top_k,
)
if get_spec().speculative_algorithm is None:
self.workspace_config = _with_early_finalize_shared_load(default_config)
self.workspace_config = _with_early_finalize_shared_load(shaped_config)
else:
# Early shared load is safe only for a single looping decode graph;
# alternating draft/verify replays can read an unfinished buffer.
@@ -142,9 +383,9 @@ class FlashInferMNNVLCuteDSLARFusion:
"CuTe DSL finalize presets on the safe (non-early-load) "
"ordering."
)
self.workspace_config = default_config
self.workspace_config = shaped_config
self.workspace = workspace_type(
tp_size=dist.get_world_size(process_group),
tp_size=self.tp_size,
tp_rank=dist.get_rank(process_group),
max_token_num=self.max_m,
hidden_dim=self.hidden_size,
@@ -166,10 +407,10 @@ class FlashInferMNNVLCuteDSLARFusion:
dist.barrier(group=process_group)
def supports(self, m: int) -> bool:
if self._destroyed or not 1 <= int(m) <= self.max_m:
if not 1 <= int(m) <= self.max_m:
return False
return self.workspace.is_buffer_size_sufficient(
tp_size=dist.get_world_size(self.process_group),
tp_size=self.tp_size,
num_tokens=int(m),
hidden_dim=self.hidden_size,
dtype=torch.bfloat16,
@@ -184,27 +425,20 @@ class FlashInferMNNVLCuteDSLARFusion:
gated_shared_output: torch.Tensor,
residual: torch.Tensor,
gamma: torch.Tensor,
norm_output: torch.Tensor | None = None,
residual_output: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
m = int(permuted_indices.shape[0])
if not self.supports(m):
raise ValueError(f"workspace does not support M={m}")
shape = (m, self.hidden_size)
if norm_output is None:
norm_output = torch.empty(shape, dtype=torch.bfloat16, device=self.device)
if residual_output is None:
residual_output = torch.empty(
shape, dtype=torch.bfloat16, device=self.device
)
norm_output = torch.empty(shape, dtype=torch.bfloat16, device=self.device)
residual_output = torch.empty(shape, dtype=torch.bfloat16, device=self.device)
pattern = self._patterns.kMoEFinalizeARResidualRMSNorm
self._allreduce_fusion(
input=routed_output,
workspace=self.workspace,
pattern=pattern,
# The public API carries the caller's PDL intent. The backend's
# routing profile owns the compiled choice and validates it.
# Caller intent only; the routing profile owns the compiled choice.
launch_with_pdl=True,
residual_in=residual,
residual_out=residual_output,
@@ -224,16 +458,12 @@ class FlashInferMNNVLCuteDSLARFusion:
local_contribution: torch.Tensor,
residual: torch.Tensor,
gamma: torch.Tensor,
norm_output: torch.Tensor | None = None,
residual_output: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
m = int(local_contribution.shape[0])
if not self.supports(m):
raise ValueError(f"workspace does not support M={m}")
if norm_output is None:
norm_output = torch.empty_like(local_contribution)
if residual_output is None:
residual_output = torch.empty_like(local_contribution)
norm_output = torch.empty_like(local_contribution)
residual_output = torch.empty_like(local_contribution)
pattern = self._patterns.kARResidualRMSNorm
self._allreduce_fusion(
@@ -250,124 +480,43 @@ class FlashInferMNNVLCuteDSLARFusion:
)
return norm_output, residual_output
def destroy(self) -> None:
if self._destroyed:
return
self.workspace.destroy()
self._destroyed = True
_WORKSPACES: dict[_WorkspaceSignature, FlashInferMNNVLCuteDSLARFusion] = {}
_WORKSPACES_LOCK = threading.RLock()
@lru_cache(maxsize=1)
def _max_workspace_instances() -> int:
from sglang.srt.environ import envs
value = int(envs.SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION_MAX_INSTANCES.get())
if value < 1:
raise ValueError(
"SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION_MAX_INSTANCES must be positive"
)
return value
_WORKSPACE: FlashInferMNNVLCuteDSLARFusion | None = None
def get_flashinfer_mnnvl_cutedsl_ar_fusion(
*,
hidden_size: int | None = None,
top_k: int | None = None,
max_m: int | None = None,
rms_epsilon: float | None = None,
weight_bias: float | None = None,
hidden_size: int,
top_k: int,
max_m: int,
rms_epsilon: float,
weight_bias: float,
) -> FlashInferMNNVLCuteDSLARFusion:
"""Lookup, or before graph capture create, the process-local workspace."""
supplied = (hidden_size, top_k, max_m, rms_epsilon, weight_bias)
if all(value is None for value in supplied):
with _WORKSPACES_LOCK:
if len(_WORKSPACES) == 1:
return next(iter(_WORKSPACES.values()))
if not _WORKSPACES:
raise RuntimeError(
"MNNVL CuTe DSL fusion workspace was not initialized before use"
)
raise RuntimeError(
"multiple MNNVL CuTe DSL fusion workspaces exist; configuration "
"arguments are required"
)
if any(value is None for value in supplied):
raise TypeError(
"hidden_size, top_k, max_m, rms_epsilon, and weight_bias must be "
"supplied together"
"""Build the process-local workspace. Must run before graph capture."""
global _WORKSPACE
if _WORKSPACE is not None:
raise RuntimeError(
"a second MNNVL CuTe DSL fusion workspace was requested; each one "
"rendezvouses its own NVLS region, and a process serves one model"
)
if not torch.cuda.is_available():
raise RuntimeError("MNNVL CuTe DSL fusion requires CUDA")
assert hidden_size is not None
assert top_k is not None
assert max_m is not None
assert rms_epsilon is not None
assert weight_bias is not None
from sglang.srt.runtime_context import get_parallel
device = torch.device("cuda", torch.cuda.current_device())
process_group = get_parallel().tp_group.device_group
domain = (
int(hidden_size),
int(top_k),
float(rms_epsilon),
float(weight_bias),
int(device.index),
id(process_group),
if torch.cuda.is_current_stream_capturing():
raise RuntimeError(
"creating an MNNVL CuTe DSL fusion workspace during CUDA Graph "
"capture is forbidden"
)
_WORKSPACE = FlashInferMNNVLCuteDSLARFusion(
hidden_size=hidden_size,
top_k=top_k,
max_m=max_m,
rms_epsilon=rms_epsilon,
weight_bias=weight_bias,
process_group=get_parallel().tp_group.device_group,
device=torch.device("cuda", torch.cuda.current_device()),
)
with _WORKSPACES_LOCK:
compatible = [
(signature.max_m, instance)
for signature, instance in _WORKSPACES.items()
if (
signature.hidden_size,
signature.top_k,
signature.rms_epsilon,
signature.weight_bias,
signature.device_index,
signature.process_group_identity,
)
== domain
and signature.max_m >= int(max_m)
]
if compatible:
return min(compatible, key=lambda item: item[0])[1]
if torch.cuda.is_current_stream_capturing():
raise RuntimeError(
"creating an MNNVL CuTe DSL fusion workspace during CUDA Graph "
"capture is forbidden"
)
if len(_WORKSPACES) >= _max_workspace_instances():
raise RuntimeError(
"MNNVL CuTe DSL fusion workspace instance limit exceeded; "
"increase SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION_MAX_INSTANCES "
"only when multiple model configurations intentionally coexist"
)
signature = _WorkspaceSignature(
hidden_size=int(hidden_size),
top_k=int(top_k),
rms_epsilon=float(rms_epsilon),
weight_bias=float(weight_bias),
max_m=int(max_m),
device_index=int(device.index),
process_group_identity=id(process_group),
)
instance = FlashInferMNNVLCuteDSLARFusion(
hidden_size=hidden_size,
top_k=top_k,
max_m=max_m,
rms_epsilon=rms_epsilon,
weight_bias=weight_bias,
process_group=process_group,
device=device,
)
_WORKSPACES[signature] = instance
return instance
return _WORKSPACE
@@ -0,0 +1,478 @@
"""FlashInfer MNNVL CuTe DSL AllReduce fusion, shared across architectures.
Two patterns share one workspace, both consumed at the next layer's input
RMSNorm: AR + residual + RMSNorm, and the same with the MoE finalize and the
shared-expert add folded in when the runner hands back a MoeFinalizeHandoff.
"""
from __future__ import annotations
import functools
import logging
from typing import Callable, Optional, Sequence
import msgspec
import torch
from sglang.srt.layers.communicator import (
CommunicateWithAllReduceAndLayerNormFn,
LayerCommunicator,
ScatterMode,
get_attn_tp_context,
)
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
from sglang.srt.layers.moe import get_moe_a2a_backend
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.runtime_context import (
cutedsl_moe_max_num_tokens,
get_disagg,
get_exec,
get_flags,
get_parallel,
)
_LayerPredicate = Callable[[torch.nn.Module], bool]
logger = logging.getLogger(__name__)
def _fused_norm_gamma(layernorm: torch.nn.Module) -> Optional[torch.Tensor]:
"""The multiplier as applied -- GemmaRMSNorm's pre-folded w + 1. None declines."""
if isinstance(layernorm, GemmaRMSNorm):
return layernorm.gemma_weight
if isinstance(layernorm, RMSNorm) and layernorm.has_weight:
return layernorm.weight
return None
def _is_supported_forward_mode(forward_mode: ForwardMode) -> bool:
return forward_mode in (
ForwardMode.DECODE,
ForwardMode.EXTEND,
ForwardMode.TARGET_VERIFY,
)
def _resolve_max_m(*, max_running_requests: int | None) -> int:
decode_config = get_exec().graph.cuda_graph_config.decode
prefill_config = get_exec().graph.cuda_graph_config.prefill
candidates = [
cutedsl_moe_max_num_tokens(),
max_running_requests,
decode_config.max_bs,
prefill_config.max_bs,
*(decode_config.bs or []),
*(prefill_config.bs or []),
]
positive = [
int(value) for value in candidates if value is not None and int(value) > 0
]
if not positive:
raise RuntimeError("framework reported no positive fusion workspace M bound")
return max(positive)
class MoeFinalizeHandoff(msgspec.Struct, frozen=True):
"""Unfinalized routed output plus the separately gated shared contribution."""
routed_output: torch.Tensor
expert_weights: torch.Tensor
permuted_indices: torch.Tensor
gated_shared_output: torch.Tensor
m: int
@classmethod
def from_flashinfer(
cls,
deferred_output,
*,
gated_shared_output: torch.Tensor,
m: int,
) -> MoeFinalizeHandoff:
top_k = int(deferred_output.top_k)
return cls(
routed_output=deferred_output.gemm2_out.view(
-1, deferred_output.gemm2_out.shape[-1]
),
expert_weights=deferred_output.expert_weights.view(-1, top_k)[:m],
permuted_indices=deferred_output.expanded_idx_to_permuted_idx.view(
-1, top_k
)[:m],
gated_shared_output=gated_shared_output,
m=int(m),
)
class CuteDSLFusionService:
def __init__(
self,
*,
hidden_size: int,
top_k: int,
rms_epsilon: float,
) -> None:
self.hidden_size = int(hidden_size)
self.top_k = int(top_k)
self.rms_epsilon = float(rms_epsilon)
self.max_m: int | None = None
self._workspace = None
def prepare(self, *, max_m: int) -> None:
if self._workspace is not None:
assert self.max_m is not None
if int(max_m) > self.max_m:
raise RuntimeError(
f"fusion workspace is already prepared for M_max={self.max_m}; "
f"refusing M_max={max_m}"
)
return
from sglang.srt.layers.flashinfer_mnnvl_cutedsl import (
get_flashinfer_mnnvl_cutedsl_ar_fusion,
)
workspace = get_flashinfer_mnnvl_cutedsl_ar_fusion(
hidden_size=self.hidden_size,
top_k=self.top_k,
max_m=int(max_m),
rms_epsilon=self.rms_epsilon,
# _fused_norm_gamma() already returns the multiplier as applied.
weight_bias=0.0,
)
self._workspace = workspace
self.max_m = workspace.max_m
def supports(self, m: int) -> bool:
"""False before prepare(), so it doubles as the readiness check."""
return self._workspace is not None and self._workspace.supports(m)
def finalize(
self,
handoff: MoeFinalizeHandoff,
residual: torch.Tensor,
gamma: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
assert self._workspace is not None
return self._workspace.moe_finalize_all_reduce_rms_norm(
routed_output=handoff.routed_output,
expert_weights=handoff.expert_weights,
permuted_indices=handoff.permuted_indices,
gated_shared_output=handoff.gated_shared_output,
residual=residual,
gamma=gamma,
)
def all_reduce_residual_rms_norm(
self,
local_contribution: torch.Tensor,
residual: torch.Tensor,
gamma: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
assert self._workspace is not None
return self._workspace.all_reduce_residual_rms_norm(
local_contribution=local_contribution,
residual=residual,
gamma=gamma,
)
class CuteDSLFusionLayerCommunicator(LayerCommunicator):
fusion_service: CuteDSLFusionService | None = None
# The runner can defer and a successor or the final norm consumes the handoff.
may_defer_moe_finalize: bool = False
# False on the last layer: the final norm does not all-reduce.
successor_absorbs_all_reduce: bool = False
# A replicated output follows the reduction; moving it would scale that by tp.
owes_local_reduction: bool = False
def prepare_attn(
self,
hidden_states,
residual,
forward_batch,
quant_format: str = "",
post_residual_addition=None,
):
if isinstance(hidden_states, MoeFinalizeHandoff):
if not self._should_use_finalize(forward_batch, hidden_states.m):
raise RuntimeError(
"received deferred MoE output on an ineligible path "
f"(M={hidden_states.m}, mode={forward_batch.forward_mode})"
)
if residual is None:
raise RuntimeError("deferred MoE finalize requires residual input")
gamma = _fused_norm_gamma(self.input_layernorm)
if gamma is None:
raise RuntimeError(
"deferred MoE finalize requires a fusable RMSNorm flavour"
)
if post_residual_addition is not None:
residual = residual + post_residual_addition
assert self.fusion_service is not None
hidden_states, residual = self.fusion_service.finalize(
handoff=hidden_states, residual=residual, gamma=gamma
)
return self._finish_prepare_attn(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
)
if (
residual is not None
and hasattr(hidden_states, "_sglang_needs_allreduce_fusion")
and hidden_states._sglang_needs_allreduce_fusion
and self._can_consume_post_moe_all_reduce(
forward_batch, int(hidden_states.shape[0])
)
):
if post_residual_addition is not None:
residual = residual + post_residual_addition
assert self.fusion_service is not None
hidden_states, residual = self.fusion_service.all_reduce_residual_rms_norm(
local_contribution=hidden_states,
residual=residual,
gamma=_fused_norm_gamma(self.input_layernorm),
)
return self._finish_prepare_attn(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
)
return super().prepare_attn(
hidden_states,
residual,
forward_batch,
quant_format=quant_format,
post_residual_addition=post_residual_addition,
)
def prepare_mlp(
self,
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
cache=None,
):
if cache is not None:
self._context.cache = cache
if self._should_use_all_reduce_rms_norm(
forward_batch, int(hidden_states.shape[0]), residual
):
assert self.fusion_service is not None and residual is not None
return self.fusion_service.all_reduce_residual_rms_norm(
local_contribution=hidden_states,
residual=residual,
gamma=_fused_norm_gamma(self.post_attention_layernorm),
)
return super().prepare_mlp(hidden_states, residual, forward_batch, cache=cache)
def _should_use_all_reduce_rms_norm(
self,
forward_batch: ForwardBatch,
m: int,
residual: Optional[torch.Tensor],
) -> bool:
communicate_fn = self._communicate_with_all_reduce_and_layer_norm_fn
if isinstance(communicate_fn, functools.partial):
norm_fn = communicate_fn.func
residual_input_mode = communicate_fn.keywords.get("residual_input_mode")
else:
norm_fn = communicate_fn
residual_input_mode = None
parallel = get_parallel()
return (
self._common_eligible(forward_batch, m)
and residual is not None
and _fused_norm_gamma(self.post_attention_layernorm) is not None
and norm_fn
is CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual
and residual_input_mode is ScatterMode.TP_ATTN_FULL
and self._context.attn_dp_size == 1
and parallel.attn_tp_size == parallel.tp_size
and not get_exec().comm.enable_quant_communications
)
def _should_use_finalize(self, forward_batch: ForwardBatch, m: int) -> bool:
return (
self._common_eligible(forward_batch, m) and get_parallel().moe_ep_size == 1
)
def _can_consume_post_moe_all_reduce(
self, forward_batch: ForwardBatch, m: int
) -> bool:
"""Incoming, and independent of this layer's own successor."""
return (
self._common_eligible(forward_batch, m)
and _fused_norm_gamma(self.input_layernorm) is not None
and not get_exec().comm.enable_quant_communications
)
def _can_absorb_post_moe_all_reduce(
self, forward_batch: ForwardBatch, m: int
) -> bool:
"""Outgoing: skip our own all-reduce because the next layer absorbs it."""
return (
self.successor_absorbs_all_reduce
and not self.owes_local_reduction
and self._can_consume_post_moe_all_reduce(forward_batch, m)
)
def should_defer_moe_finalize(
self, forward_batch: ForwardBatch, m: int | None = None
) -> bool:
"""Deferring skips the post-experts all-reduce on the promise of a handoff."""
if not self.may_defer_moe_finalize:
return False
if m is None:
m = int(forward_batch.input_ids.shape[0])
return self._should_use_finalize(forward_batch, m)
def _common_eligible(self, forward_batch: ForwardBatch, m: int) -> bool:
parallel = get_parallel()
return bool(
self.fusion_service is not None
and _is_supported_forward_mode(forward_batch.forward_mode)
and self.fusion_service.supports(m)
and not is_dp_attention_enabled()
# Also forces moe_dp_size == 1, so a hybrid EP x MoE-TP reduction
# always merges into the one TP reduction the workspace performs.
and parallel.attn_cp_size == 1
and not get_attn_tp_context().input_scattered
and get_moe_a2a_backend().is_none()
and self._context.tp_size > 1
# Restates the base's moe-cp, MOE_FULL and SCATTERED refusals.
and self.layer_scatter_modes.mlp_mode is ScatterMode.FULL
)
def should_fuse_mlp_allreduce_with_next_layer(
self, forward_batch: ForwardBatch
) -> bool:
m = int(forward_batch.input_ids.shape[0])
if self.should_defer_moe_finalize(forward_batch, m):
return True
if self._can_absorb_post_moe_all_reduce(forward_batch, m):
return True
return super().should_fuse_mlp_allreduce_with_next_layer(forward_batch)
def install_cutedsl_fusion(
layers: Sequence[torch.nn.Module],
*,
hidden_size: int,
top_k: int,
rms_epsilon: float,
can_defer_finalize: _LayerPredicate,
requires_local_reduction: _LayerPredicate | None = None,
final_norm_consumes_handoff: bool = False,
label: str,
) -> CuteDSLFusionService | None:
"""One shared workspace handle per fusion-enabled layer, or None.
Every entry of ``layers`` must carry a ``layer_communicator``.
"""
if get_flags().moe.in_speculative_scope:
# A draft shares the target's process, which holds one workspace.
return None
fusion_layers = [
layer
for layer in layers
if isinstance(layer.layer_communicator, CuteDSLFusionLayerCommunicator)
]
if not fusion_layers:
return None
# The workspace compiles one epsilon and cannot see a per-layer one.
for layer in fusion_layers:
for norm in (
layer.layer_communicator.input_layernorm,
layer.layer_communicator.post_attention_layernorm,
):
if _fused_norm_gamma(norm) is None:
continue
if float(norm.variance_epsilon) != float(rms_epsilon):
raise RuntimeError(
f"{label} CuTe DSL fusion compiles one workspace for "
f"rms_epsilon={rms_epsilon}, but a fused norm uses "
f"{norm.variance_epsilon}"
)
service = CuteDSLFusionService(
hidden_size=hidden_size,
top_k=top_k,
rms_epsilon=rms_epsilon,
)
for index, layer in enumerate(layers):
communicator = layer.layer_communicator
if not isinstance(communicator, CuteDSLFusionLayerCommunicator):
continue
successor = layers[index + 1] if index + 1 < len(layers) else None
if successor is None:
has_consumer = final_norm_consumes_handoff
else:
has_consumer = isinstance(
successor.layer_communicator, CuteDSLFusionLayerCommunicator
)
communicator.fusion_service = service
communicator.may_defer_moe_finalize = (
bool(can_defer_finalize(layer)) and has_consumer
)
communicator.successor_absorbs_all_reduce = successor is not None and (
isinstance(successor.layer_communicator, CuteDSLFusionLayerCommunicator)
)
communicator.owes_local_reduction = (
requires_local_reduction is not None and requires_local_reduction(layer)
)
logger.info(
"Installed one %s FlashInfer MNNVL CuTe DSL fusion handle for %d of %d layers "
"(%d can defer the MoE finalize)",
label,
len(fusion_layers),
len(layers),
sum(layer.layer_communicator.may_defer_moe_finalize for layer in fusion_layers),
)
return service
def prepare_cutedsl_fusion(
model: torch.nn.Module, *, max_running_requests: int | None
) -> None:
"""Build the workspace of every service installed anywhere in ``model``.
Scans the module tree, so a wrapper holding the model as a submodule (a VLM's
language_model) needs no hook of its own.
"""
communicators = [
communicator
for module in model.modules()
# Most modules carry no layer_communicator.
if isinstance(
communicator := module.__dict__.get("layer_communicator"),
CuteDSLFusionLayerCommunicator,
)
]
if not communicators or any(c.fusion_service is None for c in communicators):
raise ValueError(
"--flashinfer-allreduce-fusion-backend cutedsl is set, but "
f"{type(model).__name__} installed no CuTe DSL fusion service, so no "
"allreduce fusion would run at all. Drop the flag, or choose 'auto', "
"'trtllm' or 'mnnvl'."
)
if get_disagg().enable_pdmux:
raise RuntimeError(
"FlashInfer MNNVL CuTe DSL fusion does not support concurrent PDMux "
"streams sharing one mutable workspace"
)
services = {id(c.fusion_service): c.fusion_service for c in communicators}
max_m = _resolve_max_m(max_running_requests=max_running_requests)
for service in services.values():
service.prepare(max_m=max_m)
logger.info(
"Prepared the FlashInfer MNNVL CuTe DSL fusion workspace for M_max=%d "
"(%d fused layers)",
max_m,
len(communicators),
)
@@ -1495,13 +1495,7 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
result = trtllm_fp4_block_scale_moe(**moe_kwargs)
if defer_finalize:
gemm2_out, expert_weights, expanded_idx_to_permuted_idx = result[:3]
result = FlashInferTrtllmDeferredFinalizeOutput(
gemm2_out=gemm2_out,
expert_weights=expert_weights,
expanded_idx_to_permuted_idx=expanded_idx_to_permuted_idx,
top_k=topk_config.top_k,
)
result = _make_deferred_finalize_output(result, top_k=topk_config.top_k)
else:
result = result[0]
@@ -1,350 +0,0 @@
"""Qwen3.5 integration for FlashInfer MNNVL CuTe DSL AllReduce fusion."""
from __future__ import annotations
import logging
from dataclasses import dataclass
from typing import Optional
import torch
from sglang.srt.arg_groups.overrides import cutedsl_moe_max_num_tokens
from sglang.srt.layers.communicator import (
CommunicateWithAllReduceAndLayerNormFn,
LayerCommunicator,
ScatterMode,
get_attn_tp_context,
)
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.moe import get_moe_a2a_backend
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.runtime_context import (
get_disagg,
get_exec,
get_parallel,
)
logger = logging.getLogger(__name__)
def is_supported_forward_mode(forward_mode: ForwardMode) -> bool:
return forward_mode in (
ForwardMode.DECODE,
ForwardMode.EXTEND,
ForwardMode.TARGET_VERIFY,
)
def resolve_max_m(model_runner) -> int:
"""Use framework token bounds as the workspace-capacity source of truth."""
decode_config = get_exec().graph.cuda_graph_config.decode
prefill_config = get_exec().graph.cuda_graph_config.prefill
candidates = [
cutedsl_moe_max_num_tokens(model_runner.server_args),
model_runner.max_running_requests,
decode_config.max_bs,
prefill_config.max_bs,
*(decode_config.bs or []),
*(prefill_config.bs or []),
]
positive = [
int(value) for value in candidates if value is not None and int(value) > 0
]
if not positive:
raise RuntimeError("framework reported no positive fusion workspace M bound")
return max(positive)
@dataclass(frozen=True)
class Qwen35MoeFinalizeHandoff:
"""Unfinalized routed output plus the separately gated shared contribution."""
routed_output: torch.Tensor
expert_weights: torch.Tensor
permuted_indices: torch.Tensor
gated_shared_output: torch.Tensor
m: int
@classmethod
def from_flashinfer(
cls,
deferred_output,
*,
gated_shared_output: torch.Tensor,
m: int,
) -> Qwen35MoeFinalizeHandoff:
top_k = int(deferred_output.top_k)
return cls(
routed_output=deferred_output.gemm2_out.view(
-1, deferred_output.gemm2_out.shape[-1]
),
expert_weights=deferred_output.expert_weights.view(-1, top_k)[:m],
permuted_indices=deferred_output.expanded_idx_to_permuted_idx.view(
-1, top_k
)[:m],
gated_shared_output=gated_shared_output,
m=int(m),
)
class Qwen35FlashInferFusionService:
"""A lightweight model handle for the process-local FlashInfer workspace."""
def __init__(
self,
*,
hidden_size: int,
top_k: int,
rms_epsilon: float,
) -> None:
self.hidden_size = int(hidden_size)
self.top_k = int(top_k)
self.rms_epsilon = float(rms_epsilon)
self.max_m: int | None = None
self._workspace = None
@property
def is_prepared(self) -> bool:
return self._workspace is not None
def prepare(self, *, max_m: int) -> None:
if self._workspace is not None:
assert self.max_m is not None
if int(max_m) > self.max_m:
raise RuntimeError(
f"fusion workspace is already prepared for M_max={self.max_m}; "
f"refusing M_max={max_m}"
)
return
from sglang.srt.layers.flashinfer_mnnvl_cutedsl import (
get_flashinfer_mnnvl_cutedsl_ar_fusion,
)
workspace = get_flashinfer_mnnvl_cutedsl_ar_fusion(
hidden_size=self.hidden_size,
top_k=self.top_k,
max_m=int(max_m),
rms_epsilon=self.rms_epsilon,
# GemmaRMSNorm.gemma_weight is already checkpoint weight + 1.
weight_bias=0.0,
)
self._workspace = workspace
self.max_m = workspace.max_m
def supports(self, m: int) -> bool:
if self._workspace is None or self.max_m is None:
return False
return 1 <= int(m) <= self.max_m and self._workspace.supports(m)
def finalize(
self,
handoff: Qwen35MoeFinalizeHandoff,
residual: torch.Tensor,
gamma: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
self._validate_finalize(handoff, residual, gamma)
assert self._workspace is not None
return self._workspace.moe_finalize_all_reduce_rms_norm(
routed_output=handoff.routed_output,
expert_weights=handoff.expert_weights,
permuted_indices=handoff.permuted_indices,
gated_shared_output=handoff.gated_shared_output,
residual=residual,
gamma=gamma,
)
def all_reduce_residual_rms_norm(
self,
local_contribution: torch.Tensor,
residual: torch.Tensor,
gamma: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
self._validate_matrix(local_contribution, "local_contribution")
self._validate_matrix(residual, "residual", m=local_contribution.shape[0])
self._validate_gamma(gamma)
if not self.supports(local_contribution.shape[0]):
raise ValueError(f"unsupported M={local_contribution.shape[0]}")
assert self._workspace is not None
return self._workspace.all_reduce_residual_rms_norm(
local_contribution=local_contribution,
residual=residual,
gamma=gamma,
)
def _validate_finalize(
self,
handoff: Qwen35MoeFinalizeHandoff,
residual: torch.Tensor,
gamma: torch.Tensor,
) -> None:
if not self.supports(handoff.m):
raise ValueError(f"unsupported M={handoff.m}")
self._validate_matrix(handoff.routed_output, "routed_output", exact_m=False)
expected_metadata = (handoff.m, self.top_k)
if tuple(handoff.expert_weights.shape) != expected_metadata:
raise ValueError("expert_weights must have shape [M, top_k]")
if tuple(handoff.permuted_indices.shape) != expected_metadata:
raise ValueError("permuted_indices must have shape [M, top_k]")
if handoff.expert_weights.dtype != torch.bfloat16:
raise ValueError("expert_weights must be BF16")
if handoff.permuted_indices.dtype != torch.int32:
raise ValueError("permuted_indices must be Int32")
self._validate_matrix(
handoff.gated_shared_output, "gated_shared_output", m=handoff.m
)
self._validate_matrix(residual, "residual", m=handoff.m)
self._validate_gamma(gamma)
def _validate_matrix(
self,
tensor: torch.Tensor,
name: str,
*,
m: int | None = None,
exact_m: bool = True,
) -> None:
if tensor.ndim != 2 or tensor.shape[1] != self.hidden_size:
raise ValueError(f"{name} must have shape [M, hidden_size]")
if m is not None and exact_m and tensor.shape[0] != int(m):
raise ValueError(f"{name} has the wrong M dimension")
if tensor.dtype != torch.bfloat16 or not tensor.is_contiguous():
raise ValueError(f"{name} must be contiguous BF16")
def _validate_gamma(self, gamma: torch.Tensor) -> None:
if (
tuple(gamma.shape) != (self.hidden_size,)
or gamma.dtype != torch.bfloat16
or not gamma.is_contiguous()
):
raise ValueError("gamma must be contiguous BF16 [hidden_size]")
class Qwen35FlashInferLayerCommunicator(LayerCommunicator):
"""Qwen-only hooks; generic LayerCommunicator remains backend agnostic."""
fusion_service: Qwen35FlashInferFusionService | None = None
def prepare_attn(
self,
hidden_states,
residual,
forward_batch,
quant_format: str = "",
post_residual_addition=None,
):
if isinstance(hidden_states, Qwen35MoeFinalizeHandoff):
if not self.should_use_finalize(forward_batch, hidden_states.m):
raise RuntimeError("received deferred MoE output on an ineligible path")
if residual is None:
raise RuntimeError("deferred MoE finalize requires residual input")
if not hasattr(self.input_layernorm, "gemma_weight"):
raise RuntimeError("deferred Qwen finalize requires GemmaRMSNorm")
if post_residual_addition is not None:
residual = residual + post_residual_addition
assert self.fusion_service is not None
return self.fusion_service.finalize(
hidden_states, residual, self.input_layernorm.gemma_weight
)
return super().prepare_attn(
hidden_states,
residual,
forward_batch,
quant_format=quant_format,
post_residual_addition=post_residual_addition,
)
def prepare_mlp(
self,
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
cache=None,
):
if cache is not None:
self._context.cache = cache
if self.should_use_all_reduce_rms_norm(
forward_batch, int(hidden_states.shape[0]), residual
):
assert self.fusion_service is not None and residual is not None
return self.fusion_service.all_reduce_residual_rms_norm(
hidden_states,
residual,
self.post_attention_layernorm.gemma_weight,
)
return super().prepare_mlp(hidden_states, residual, forward_batch, cache=cache)
def should_use_all_reduce_rms_norm(
self,
forward_batch: ForwardBatch,
m: int,
residual: Optional[torch.Tensor],
) -> bool:
communicate_fn = self._communicate_with_all_reduce_and_layer_norm_fn
norm_fn = getattr(communicate_fn, "func", communicate_fn)
residual_input_mode = getattr(communicate_fn, "keywords", {}).get(
"residual_input_mode"
)
parallel = get_parallel()
return (
self._common_eligible(forward_batch, m)
and residual is not None
and hasattr(self.post_attention_layernorm, "gemma_weight")
and norm_fn
is CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual
and residual_input_mode is ScatterMode.TP_ATTN_FULL
and self._context.attn_dp_size == 1
and parallel.attn_tp_size == parallel.tp_size
and not get_exec().comm.enable_quant_communications
)
def should_use_finalize(self, forward_batch: ForwardBatch, m: int) -> bool:
parallel = get_parallel()
return (
self._common_eligible(forward_batch, m)
and self.layer_scatter_modes.mlp_mode is not ScatterMode.SCATTERED
and parallel.moe_ep_size == 1
)
def _common_eligible(self, forward_batch: ForwardBatch, m: int) -> bool:
parallel = get_parallel()
return bool(
self.fusion_service is not None
and self.fusion_service.is_prepared
and is_supported_forward_mode(forward_batch.forward_mode)
and self.fusion_service.supports(m)
and not is_dp_attention_enabled()
and parallel.attn_cp_size == 1
and not get_attn_tp_context().input_scattered
and get_moe_a2a_backend().is_none()
and self._context.tp_size > 1
)
def should_fuse_mlp_allreduce_with_next_layer(
self, forward_batch: ForwardBatch
) -> bool:
m = (
int(forward_batch.input_ids.shape[0])
if getattr(forward_batch, "input_ids", None) is not None
else 0
)
if self.should_use_finalize(forward_batch, m):
# The Qwen model consumes the final layer's handoff with its final
# GemmaRMSNorm, so this is intentionally also true for that layer.
return True
return super().should_fuse_mlp_allreduce_with_next_layer(forward_batch)
def prepare_qwen35_flashinfer_fusion(model, model_runner) -> None:
service = getattr(model, "flashinfer_mnnvl_cutedsl_fusion", None)
if service is None:
return
if get_disagg().enable_pdmux:
raise RuntimeError(
"FlashInfer MNNVL CuTe DSL fusion does not support concurrent PDMux "
"streams sharing one mutable workspace"
)
service.prepare(max_m=resolve_max_m(model_runner))
logger.info(
"Prepared Qwen3.5 FlashInfer MNNVL CuTe DSL fusion workspace for M_max=%d",
service.max_m,
)
@@ -288,11 +288,29 @@ class BaseRunner(ABC):
with custom_all_reduce.register_graph_buffers).
"""
mr = self.model_runner
from sglang.srt.layers.communicator import FUSE_ALLREDUCE_MAX_BATCH_SIZE
from sglang.srt.layers.flashinfer_comm_fusion import (
pre_initialize_workspaces,
resolve_flashinfer_allreduce_fusion_backend,
uses_cutedsl_ar_fusion,
)
if get_exec().comm.flashinfer_allreduce_fusion_backend is None:
return
from sglang.srt.layers.communicator import FUSE_ALLREDUCE_MAX_BATCH_SIZE
from sglang.srt.layers.flashinfer_comm_fusion import pre_initialize_workspaces
if uses_cutedsl_ar_fusion():
# Nothing else resolves the configured backend, so check the platform.
resolve_flashinfer_allreduce_fusion_backend()
if not mr.is_draft_worker:
# A draft installs no fusion communicator.
from sglang.srt.layers.moe.cutedsl_ar_fusion import (
prepare_cutedsl_fusion,
)
prepare_cutedsl_fusion(
mr.model, max_running_requests=mr.max_running_requests
)
return
pre_initialize_workspaces(
max_token_num=FUSE_ALLREDUCE_MAX_BATCH_SIZE,
+66 -5
View File
@@ -81,6 +81,7 @@ from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
from sglang.srt.layers.dcp.planner import (
prepare_decode_context_parallel_metadata,
)
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
@@ -575,6 +576,9 @@ class DeepseekV2MoE(nn.Module):
self.moe_ep_size = get_parallel().moe_ep_size
self.routed_scaling_factor = config.routed_scaling_factor
self.n_shared_experts = config.n_shared_experts
self._deferred_finalize_max_tokens = (
envs.SGLANG_MOE_DEFERRED_FINALIZE_MAX_TOKENS.get()
)
n_shared_experts = (
0 if config.n_shared_experts is None else int(config.n_shared_experts)
@@ -1061,6 +1065,10 @@ class DeepseekV2MoE(nn.Module):
and not self._shared_expert_tp1
and topk_output.format == TopKOutputFormat.BYPASSED
and self.experts.supports_deferred_finalize
and (
self._deferred_finalize_max_tokens <= 0
or hidden_states.shape[0] <= self._deferred_finalize_max_tokens
)
)
if deferred_finalize:
final_hidden_states = self.experts.forward_deferred_finalize(
@@ -1094,6 +1102,17 @@ class DeepseekV2MoE(nn.Module):
# The routed-input pre-quant was already joined inside the routed MoE apply.
current_stream.wait_stream(self.alt_stream)
if deferred_finalize and get_forward().defer_moe_finalize:
# deferred_finalize excludes _shared_expert_tp1, so the shared add folds in.
assert shared_output is not None
from sglang.srt.layers.moe.cutedsl_ar_fusion import MoeFinalizeHandoff
return MoeFinalizeHandoff.from_flashinfer(
final_hidden_states,
gated_shared_output=shared_output,
m=hidden_states.shape[0],
)
all_reduce_done = False
if deferred_finalize:
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
@@ -2482,6 +2501,10 @@ class DeepseekV2AttentionMLA(
return quant_config
def _use_mnnvl_cutedsl_fusion() -> bool:
return _is_cuda and uses_cutedsl_ar_fusion()
class DeepseekV2DecoderLayer(nn.Module):
def __init__(
self,
@@ -2586,11 +2609,17 @@ class DeepseekV2DecoderLayer(nn.Module):
self._gfx95_quant_format = self._detect_gfx95_quant_format()
communicator_cls = (
DSACPLayerCommunicator
if get_parallel().enable_prefill_cp
else LayerCommunicator
)
if get_parallel().enable_prefill_cp:
communicator_cls = DSACPLayerCommunicator
elif not is_nextn and _use_mnnvl_cutedsl_fusion():
# Dense layers too: selecting cutedsl turns the legacy fusion off.
from sglang.srt.layers.moe.cutedsl_ar_fusion import (
CuteDSLFusionLayerCommunicator,
)
communicator_cls = CuteDSLFusionLayerCommunicator
else:
communicator_cls = LayerCommunicator
self.layer_communicator = communicator_cls(
layer_scatter_modes=self.layer_scatter_modes,
input_layernorm=self.input_layernorm,
@@ -2696,6 +2725,8 @@ class DeepseekV2DecoderLayer(nn.Module):
isinstance(self.mlp, DeepseekV2MoE)
and not self.mlp.experts.moe_runner_config.inplace
and not torch.compiler.is_compiling()
# A deferred MoE finalize handoff from the previous layer is no buffer.
and isinstance(hidden_states_orig, torch.Tensor)
):
from sglang.srt.layers.moe.moe_runner.base import moe_output_buffer_ctx
@@ -2917,6 +2948,34 @@ class DeepseekV2Model(nn.Module):
# llama_4_scaling: for supporting Mistral-Large-3 model
self.llama_4_scaling_config = getattr(config, "llama_4_scaling", None)
from sglang.srt.layers.moe.cutedsl_ar_fusion import install_cutedsl_fusion
install_cutedsl_fusion(
# PP pads self.layers with PPMissingLayer, which has no communicator.
# A stage's last layer has no successor, so it keeps its reduction.
self.layers[self.start_layer : self.end_layer],
hidden_size=config.hidden_size,
top_k=config.num_experts_per_tok,
rms_epsilon=config.rms_norm_eps,
# Only the dual-stream path hands off, which needs an unfused shared
# expert; with V3/R1's default shared-experts fusion only AR + norm fuse.
# A TP1 shared expert is added after the all-reduce and cannot move it.
can_defer_finalize=lambda layer: (
isinstance(layer.mlp, DeepseekV2MoE)
and layer.mlp.experts.supports_deferred_finalize
and layer.mlp.num_fused_shared_experts == 0
and not layer.mlp._shared_expert_tp1
),
# A TP1-replicated output must not ride into the next layer's reduction.
requires_local_reduction=lambda layer: (
layer.mlp._shared_expert_tp1
if isinstance(layer.mlp, DeepseekV2MoE)
else layer.mlp.tp_size == 1
),
final_norm_consumes_handoff=False,
label="DeepSeek-V3/GLM",
)
def get_input_embeddings(self) -> torch.Tensor:
return self.embed_tokens
@@ -3385,6 +3444,8 @@ def dsv2_flashinfer_moe_dual_stream_graph(
fuse_mlp_allreduce=fuse_mlp_allreduce,
mlp_reduce_scatter=mlp_reduce_scatter,
flashinfer_trtllm_bypass=True,
# The op's Tensor schema cannot carry a MoeFinalizeHandoff.
defer_moe_finalize=False,
):
return moe_fusion.forward_normal_dual_stream(hidden_states)
+4 -6
View File
@@ -51,6 +51,7 @@ from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
@@ -337,8 +338,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
num_fused_shared_experts=self.num_fused_shared_experts,
inplace=not _needs_hidden_after_experts,
enable_qwen35_fp8_deferred_finalize=(
config.model_type == "qwen3_5_moe_text"
and envs.SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION.get()
config.model_type == "qwen3_5_moe_text" and uses_cutedsl_ar_fusion()
),
)
@@ -865,11 +865,9 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
if defer_finalize:
if shared_output is None:
raise RuntimeError("Qwen deferred finalize requires shared output")
from sglang.srt.layers.moe.qwen35_flashinfer_fusion import (
Qwen35MoeFinalizeHandoff,
)
from sglang.srt.layers.moe.cutedsl_ar_fusion import MoeFinalizeHandoff
return Qwen35MoeFinalizeHandoff.from_flashinfer(
return MoeFinalizeHandoff.from_flashinfer(
final_hidden_states,
gated_shared_output=shared_output,
m=num_tokens,
+28 -90
View File
@@ -47,6 +47,7 @@ from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
# Layers - Others
from sglang.srt.layers.layernorm import GemmaRMSNorm
@@ -172,10 +173,7 @@ def _disable_shared_experts_fusion() -> bool:
# intent through the accessor's fallback.
# The deferred-finalize ABI needs the shared expert as a separate, gated
# local contribution; it cannot consume a shared slot fused into routed MoE.
return bool(
envs.SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION.get()
or is_shared_experts_fusion_disabled()
)
return bool(uses_cutedsl_ar_fusion() or is_shared_experts_fusion_disabled())
def _maybe_enable_silu_fp4_quant_fusion(mlp: nn.Module) -> None:
@@ -209,17 +207,17 @@ def _use_mnnvl_cutedsl_fusion(config: Qwen3_5TextConfig, is_nextn: bool) -> bool
return bool(
not is_nextn
and config.model_type == "qwen3_5_moe_text"
and envs.SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION.get()
and uses_cutedsl_ar_fusion()
)
def _layer_communicator_class(config: Qwen3_5TextConfig, is_nextn: bool):
if _use_mnnvl_cutedsl_fusion(config, is_nextn):
from sglang.srt.layers.moe.qwen35_flashinfer_fusion import (
Qwen35FlashInferLayerCommunicator,
from sglang.srt.layers.moe.cutedsl_ar_fusion import (
CuteDSLFusionLayerCommunicator,
)
return Qwen35FlashInferLayerCommunicator
return CuteDSLFusionLayerCommunicator
return LayerCommunicator
@@ -305,22 +303,15 @@ def _select_fused_ar_input_for_linear(hidden_states, linear: nn.Module):
def _finish_mlp_output(hidden_states, *, expect_deferred: bool):
if not expect_deferred:
if not isinstance(hidden_states, torch.Tensor):
from sglang.srt.layers.moe.qwen35_flashinfer_fusion import (
Qwen35MoeFinalizeHandoff,
)
from sglang.srt.layers.moe.cutedsl_ar_fusion import MoeFinalizeHandoff
if isinstance(hidden_states, Qwen35MoeFinalizeHandoff):
raise RuntimeError("unexpected deferred-finalize handoff")
if not expect_deferred:
if isinstance(hidden_states, MoeFinalizeHandoff):
raise RuntimeError("unexpected deferred-finalize handoff")
hidden_states._sglang_needs_allreduce_fusion = True
return hidden_states
from sglang.srt.layers.moe.qwen35_flashinfer_fusion import (
Qwen35MoeFinalizeHandoff,
)
if not isinstance(hidden_states, Qwen35MoeFinalizeHandoff):
if not isinstance(hidden_states, MoeFinalizeHandoff):
raise RuntimeError("Qwen3.5 expected a FlashInfer deferred-finalize handoff")
return hidden_states
@@ -1199,8 +1190,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
fuse_mlp_allreduce
and isinstance(hidden_states, torch.Tensor)
and isinstance(self.mlp, Qwen2MoeSparseMoeBlock)
and hasattr(self.layer_communicator, "should_use_finalize")
and self.layer_communicator.should_use_finalize(
and self.layer_communicator.should_defer_moe_finalize(
forward_batch, int(hidden_states.shape[0])
)
)
@@ -1637,8 +1627,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
fuse_mlp_allreduce
and isinstance(hidden_states, torch.Tensor)
and isinstance(self.mlp, Qwen2MoeSparseMoeBlock)
and hasattr(self.layer_communicator, "should_use_finalize")
and self.layer_communicator.should_use_finalize(
and self.layer_communicator.should_defer_moe_finalize(
forward_batch, int(hidden_states.shape[0])
)
)
@@ -1825,26 +1814,20 @@ class Qwen3_5ForCausalLM(nn.Module):
"layers: "
f"{unsupported_layers}"
)
from sglang.srt.layers.moe.qwen35_flashinfer_fusion import (
Qwen35FlashInferFusionService,
Qwen35FlashInferLayerCommunicator,
from sglang.srt.layers.moe.cutedsl_ar_fusion import (
install_cutedsl_fusion,
)
self.flashinfer_mnnvl_cutedsl_fusion = Qwen35FlashInferFusionService(
self.flashinfer_mnnvl_cutedsl_fusion = install_cutedsl_fusion(
self.layers,
hidden_size=config.hidden_size,
top_k=config.num_experts_per_tok,
rms_epsilon=config.rms_norm_eps,
)
for layer in self.layers:
communicator = layer.layer_communicator
if not isinstance(communicator, Qwen35FlashInferLayerCommunicator):
raise RuntimeError(
"Qwen3.5 fusion-enabled layer has the wrong communicator"
)
communicator.fusion_service = self.flashinfer_mnnvl_cutedsl_fusion
logger.info(
"Installed one Qwen3.5 FlashInfer fusion handle for %d layers",
len(self.layers),
# Every layer was checked above.
can_defer_finalize=lambda layer: True,
# The final GemmaRMSNorm closes out the last layer's handoff.
final_norm_consumes_handoff=True,
label="Qwen3.5",
)
# Final normalization
@@ -1879,13 +1862,6 @@ class Qwen3_5ForCausalLM(nn.Module):
logger.info(
"Packed BF16/FP8 GDN input projection enabled for %d layers", packed
)
if self.flashinfer_mnnvl_cutedsl_fusion is None:
return
from sglang.srt.layers.moe.qwen35_flashinfer_fusion import (
prepare_qwen35_flashinfer_fusion,
)
prepare_qwen35_flashinfer_fusion(self, model_runner)
def set_dflash_layers_to_capture(self, layers_to_capture: list[int]):
self.layers_to_capture = layers_to_capture
@@ -1972,61 +1948,23 @@ class Qwen3_5ForCausalLM(nn.Module):
)
# The final layer has no successor to consume its deferred MoE tail.
trace_final_norm = envs.SGLANG_TRACE_QWEN35_FINAL_NORM.get()
use_native_final_norm = envs.SGLANG_QWEN35_NATIVE_FINAL_NORM.get()
is_deferred_finalize = False
if self.flashinfer_mnnvl_cutedsl_fusion is not None:
from sglang.srt.layers.moe.qwen35_flashinfer_fusion import (
Qwen35MoeFinalizeHandoff,
)
from sglang.srt.layers.moe.cutedsl_ar_fusion import MoeFinalizeHandoff
is_deferred_finalize = isinstance(hidden_states, Qwen35MoeFinalizeHandoff)
is_deferred_finalize = isinstance(hidden_states, MoeFinalizeHandoff)
if is_deferred_finalize:
if residual is None or self.flashinfer_mnnvl_cutedsl_fusion is None:
if residual is None:
raise RuntimeError("invalid final deferred MoE handoff")
hidden_states, _ = self.flashinfer_mnnvl_cutedsl_fusion.finalize(
hidden_states, residual, self.norm.gemma_weight
handoff=hidden_states, residual=residual, gamma=self.norm.gemma_weight
)
elif hidden_states.shape[0] != 0:
if trace_final_norm:
print(
"SGLANG_TRACE_QWEN35_FINAL_NORM "
f"stage=pre_sync_enter hidden={tuple(hidden_states.shape)} "
f"hidden_stride={hidden_states.stride()} "
f"hidden_dtype={hidden_states.dtype} "
f"hidden_contiguous={hidden_states.is_contiguous()} "
f"residual={None if residual is None else tuple(residual.shape)} "
f"native={use_native_final_norm}",
flush=True,
)
torch.cuda.synchronize()
print(
"SGLANG_TRACE_QWEN35_FINAL_NORM stage=pre_sync_returned",
flush=True,
)
if residual is None:
hidden_states = (
self.norm.forward_native(hidden_states)
if use_native_final_norm
else self.norm(hidden_states)
)
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = (
self.norm.forward_native(hidden_states, residual)
if use_native_final_norm
else self.norm(hidden_states, residual)
)
if trace_final_norm:
print(
"SGLANG_TRACE_QWEN35_FINAL_NORM stage=post_sync_enter",
flush=True,
)
torch.cuda.synchronize()
print(
"SGLANG_TRACE_QWEN35_FINAL_NORM stage=post_sync_returned",
flush=True,
)
hidden_states, _ = self.norm(hidden_states, residual)
if len(aux_hidden_states) == 0:
return hidden_states
+3
View File
@@ -660,9 +660,11 @@ class ForwardFlags:
# fuse_mlp_allreduce: next residual+LN absorbs the post-MLP all-reduce.
# mlp_reduce_scatter: postprocess will reduce-scatter (skip MLP AR).
# flashinfer_trtllm_bypass: deepseek dual-stream graph topk bypass.
# defer_moe_finalize: next layer's fused collective absorbs the finalize.
"fuse_mlp_allreduce": False,
"mlp_reduce_scatter": False,
"flashinfer_trtllm_bypass": False,
"defer_moe_finalize": False,
# LayerNorm sequence parallelism region; see layers/layernorm_sp.py.
"sp_active": False,
}
@@ -679,6 +681,7 @@ class ForwardFlags:
"fuse_mlp_allreduce",
"mlp_reduce_scatter",
"flashinfer_trtllm_bypass",
"defer_moe_finalize",
"sp_active",
}
)