Refactor the Cute-DSL AR fusion to support DeepseekV2 archs (GLM-5.3, etc.) (#39816)
Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai> Co-authored-by: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com>
This commit is contained in:
co-authored by
Mohammad Angkad
Mohammad Miadh Angkad
parent
ce06a14444
commit
3177d10ca6
@@ -630,7 +630,7 @@ class ExecComm(msgspec.Struct):
|
||||
"Enforce disable FlashInfer allreduce fusion.",
|
||||
] = False
|
||||
flashinfer_allreduce_fusion_backend: A[
|
||||
Optional[Literal["auto", "trtllm", "mnnvl"]],
|
||||
Optional[Literal["auto", "trtllm", "mnnvl", "cutedsl"]],
|
||||
Arg(
|
||||
help=(
|
||||
"Enable FlashInfer allreduce fusion and choose backend. "
|
||||
@@ -641,6 +641,9 @@ class ExecComm(msgspec.Struct):
|
||||
"'trtllm': available on single-node systems only. "
|
||||
"'mnnvl': available on SM90 single-node systems and SM100/SM103 "
|
||||
"single-node or multi-node systems via MNNVL fabric. "
|
||||
"'cutedsl': Blackwell-only bf16 MNNVL CuTe DSL backend; also "
|
||||
"fuses the MoE finalize and the shared-expert add into the "
|
||||
"collective when the MoE runner can defer them. "
|
||||
"Fuses allreduce with Residual + RMSNorm for supported MoE models."
|
||||
),
|
||||
resolvable=True,
|
||||
|
||||
@@ -202,6 +202,7 @@ def validate_flashinfer_megamoe_model(server_args: Any) -> None:
|
||||
"DeepseekV32ForCausalLM",
|
||||
"DeepseekV4ForCausalLM",
|
||||
"Glm4MoeForCausalLM",
|
||||
"GlmMoeDsaForCausalLM",
|
||||
"NemotronHForCausalLM",
|
||||
"NemotronHPuzzleForCausalLM",
|
||||
"Qwen2MoeForCausalLM",
|
||||
|
||||
@@ -920,20 +920,6 @@ def _flashinfer_allreduce_fusion_auto_enable(view: Any) -> dict:
|
||||
and view.nnodes == 1
|
||||
and not view.disable_custom_all_reduce
|
||||
)
|
||||
if envs.SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION.get() and model_arch in {
|
||||
"Qwen3_5MoeForCausalLM",
|
||||
"Qwen3_5MoeForConditionalGeneration",
|
||||
}:
|
||||
# The Qwen backend owns one workspace for ordinary AR and MoE finalize;
|
||||
# do not allocate or fall back to the legacy TRTLLM/MNNVL workspace.
|
||||
if view.flashinfer_allreduce_fusion_backend is not None:
|
||||
logger.warning(
|
||||
"SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION owns both Qwen3.5 "
|
||||
"AllReduce fusion patterns; suppressing the separately configured "
|
||||
"--flashinfer-allreduce-fusion-backend=%s",
|
||||
view.flashinfer_allreduce_fusion_backend,
|
||||
)
|
||||
return {"flashinfer_allreduce_fusion_backend": None}
|
||||
if (
|
||||
view.flashinfer_allreduce_fusion_backend is None
|
||||
and model_arch in _FLASHINFER_ALLREDUCE_FUSION_ARCHS
|
||||
|
||||
@@ -236,8 +236,12 @@ def _set_all_reduce_flags() -> None:
|
||||
set_custom_all_reduce(not get_exec().comm.disable_custom_all_reduce)
|
||||
set_mscclpp_all_reduce(get_exec().comm.enable_mscclpp)
|
||||
set_torch_symm_mem_all_reduce(get_exec().comm.enable_torch_symm_mem)
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
|
||||
|
||||
set_flashinfer_allreduce_only(
|
||||
get_exec().comm.flashinfer_allreduce_fusion_backend is not None
|
||||
# cutedsl has no legacy workspace for tagged groups to reduce over.
|
||||
and not uses_cutedsl_ar_fusion()
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1244,6 +1244,9 @@ class Envs:
|
||||
SGLANG_NIXL_EP_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(128)
|
||||
SGLANG_PPLX_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(128)
|
||||
SGLANG_ENABLE_MOE_DEFERRED_FINALIZE = EnvBool(True)
|
||||
# The [M*top_k, hidden] HBM round trip pays only at small M; 0 disables.
|
||||
# GLM-5.2 (H=6144): neutral at 192 on B300 TP8, crossover 192-223 on GB300 TP4.
|
||||
SGLANG_MOE_DEFERRED_FINALIZE_MAX_TOKENS = EnvInt(192)
|
||||
# DeepSeek/GLM MoE (deepseek_v2.py): quantize the (dp-gathered) MoE input
|
||||
# to per-token-group-128 fp8 ONCE and feed both the fused shared-expert
|
||||
# GEMM (cutlass w8a8 linear) and the routed experts' triton fused runner,
|
||||
@@ -1769,13 +1772,6 @@ class Envs:
|
||||
|
||||
# Qwen3.5 and GDN
|
||||
SGLANG_ENABLE_GDN_DECODE_FUSED_PROJ_CONV = EnvBool(True)
|
||||
SGLANG_TRACE_QWEN35_FINAL_NORM = EnvBool(False)
|
||||
SGLANG_QWEN35_NATIVE_FINAL_NORM = EnvBool(False)
|
||||
# One switch enables deferred MoE finalize and AR + residual + RMSNorm.
|
||||
SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION = EnvBool(False)
|
||||
# Distinct workspace configurations allowed in one process. Production
|
||||
# uses one model/configuration per rank, so fail closed on accidental reuse.
|
||||
SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION_MAX_INSTANCES = EnvInt(1)
|
||||
|
||||
# ===================================================================
|
||||
# Plugin system
|
||||
@@ -1887,10 +1883,22 @@ class _DeprecatedEnv:
|
||||
# ad-hoc warnings. For a rename where the old name must keep working through a
|
||||
# descriptor, use EnvBoolWithAlias / EnvIntWithAlias instead.
|
||||
_DEPRECATED_ENVS: Dict[str, _DeprecatedEnv] = {
|
||||
"SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION": _DeprecatedEnv(
|
||||
note=(
|
||||
"Pass --flashinfer-allreduce-fusion-backend cutedsl instead. "
|
||||
"Without it an eligible model auto-enables the legacy mnnvl "
|
||||
"backend rather than the CuTe DSL fusion."
|
||||
)
|
||||
),
|
||||
"SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION_MAX_INSTANCES": _DeprecatedEnv(
|
||||
note="One workspace per process is now an invariant, not a limit."
|
||||
),
|
||||
# Removed without replacement.
|
||||
"SGLANG_ENABLE_CP_V2": _DeprecatedEnv(
|
||||
note="Strategy-based prefill context parallelism is now the only generic implementation."
|
||||
),
|
||||
"SGLANG_TRACE_QWEN35_FINAL_NORM": _DeprecatedEnv(),
|
||||
"SGLANG_QWEN35_NATIVE_FINAL_NORM": _DeprecatedEnv(),
|
||||
"SGLANG_ENABLE_HICACHE_BUFFER_ANCHOR_LOCK": _DeprecatedEnv(
|
||||
note="Buffer-mode anchor pinning is always on; set "
|
||||
"SGLANG_HICACHE_BUFFER_ANCHOR_LOCK_CAP=0 to disable it."
|
||||
|
||||
@@ -58,7 +58,10 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_enable_moe_cp_allgather,
|
||||
moe_cp_all_gather_into_tensor,
|
||||
)
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import is_flashinfer_allreduce_unavailable
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import (
|
||||
is_flashinfer_allreduce_unavailable,
|
||||
uses_cutedsl_ar_fusion,
|
||||
)
|
||||
from sglang.srt.layers.moe import (
|
||||
can_merge_post_experts_all_reduce,
|
||||
deferred_post_experts_all_reduce,
|
||||
@@ -198,6 +201,8 @@ def apply_flashinfer_allreduce_fusion(batch_size: int):
|
||||
and _is_flashinfer_available
|
||||
and not is_dp_attention_enabled()
|
||||
and get_exec().comm.flashinfer_allreduce_fusion_backend is not None
|
||||
# cutedsl runs its own fused path from the fusion communicator.
|
||||
and not uses_cutedsl_ar_fusion()
|
||||
and not is_flashinfer_allreduce_unavailable()
|
||||
# Symbolic size checks stay last: under Dynamo tracing they guard on
|
||||
# the dynamic token dim, so statically-off configs must short-circuit
|
||||
@@ -859,6 +864,14 @@ class LayerCommunicator:
|
||||
post_residual_addition,
|
||||
)
|
||||
|
||||
return self._finish_prepare_attn(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
def _finish_prepare_attn(self, hidden_states, residual, forward_batch):
|
||||
"""Tail every prepare_attn path must run, or ``attn_inputs`` is unset."""
|
||||
hidden_states = self._communicate_simple_fn(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
@@ -953,6 +966,12 @@ class LayerCommunicator:
|
||||
return True
|
||||
return False
|
||||
|
||||
def should_defer_moe_finalize(
|
||||
self, forward_batch: ForwardBatch, m: int | None = None
|
||||
) -> bool:
|
||||
"""Whether the MoE may hand an unfinalized output to the next layer."""
|
||||
return False
|
||||
|
||||
# NOTE: This function will cause torch recompilation
|
||||
def should_fuse_mlp_allreduce_with_next_layer(
|
||||
self, forward_batch: ForwardBatch
|
||||
@@ -1046,11 +1065,13 @@ def scatter_mode_layouts(
|
||||
|
||||
class FfnExit:
|
||||
"""One FFN's reduction decision. Inside the ``with`` block it is published as
|
||||
``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` on ``get_forward()``."""
|
||||
``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` / ``defer_moe_finalize`` on
|
||||
``get_forward()``."""
|
||||
|
||||
__slots__ = (
|
||||
"communicator",
|
||||
"forward_batch",
|
||||
"defer_moe_finalize",
|
||||
"fuse_mlp_allreduce",
|
||||
"mlp_reduce_scatter",
|
||||
"_scope",
|
||||
@@ -1059,13 +1080,17 @@ class FfnExit:
|
||||
def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch):
|
||||
self.communicator = communicator
|
||||
self.forward_batch = forward_batch
|
||||
self.defer_moe_finalize = communicator.should_defer_moe_finalize(forward_batch)
|
||||
# Deferring implies fusing: a handoff skips the post-experts all-reduce.
|
||||
self.fuse_mlp_allreduce = (
|
||||
communicator.should_fuse_mlp_allreduce_with_next_layer(forward_batch)
|
||||
self.defer_moe_finalize
|
||||
or communicator.should_fuse_mlp_allreduce_with_next_layer(forward_batch)
|
||||
)
|
||||
self.mlp_reduce_scatter = communicator.should_use_reduce_scatter(forward_batch)
|
||||
self._scope = get_forward().scoped(
|
||||
fuse_mlp_allreduce=self.fuse_mlp_allreduce,
|
||||
mlp_reduce_scatter=self.mlp_reduce_scatter,
|
||||
defer_moe_finalize=self.defer_moe_finalize,
|
||||
)
|
||||
|
||||
def __enter__(self) -> "FfnExit":
|
||||
@@ -1079,6 +1104,10 @@ class FfnExit:
|
||||
self, hidden_states: torch.Tensor, residual: torch.Tensor
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Leave the reduction to the next layer's input norm, or postprocess."""
|
||||
if not isinstance(hidden_states, torch.Tensor):
|
||||
# A deferred MoE finalize handoff, consumed by the next prepare_attn.
|
||||
assert self.defer_moe_finalize, "unrequested deferred MoE handoff"
|
||||
return hidden_states, residual
|
||||
if self.fuse_mlp_allreduce:
|
||||
hidden_states._sglang_needs_allreduce_fusion = True
|
||||
return hidden_states, residual
|
||||
|
||||
@@ -47,6 +47,14 @@ def _resolve_backend(backend: str, is_multi_node: bool = False) -> str:
|
||||
"FlashInfer allreduce fusion requires SM90 or SM10X NVIDIA GPUs."
|
||||
)
|
||||
|
||||
if backend == "cutedsl":
|
||||
if not get_platform().is_sm100:
|
||||
raise ValueError(
|
||||
"FlashInfer allreduce fusion cutedsl backend requires a "
|
||||
"Blackwell system."
|
||||
)
|
||||
return backend
|
||||
|
||||
if backend == "auto":
|
||||
if is_multi_node:
|
||||
if get_platform().is_sm100:
|
||||
@@ -72,6 +80,11 @@ def _resolve_backend(backend: str, is_multi_node: bool = False) -> str:
|
||||
return backend
|
||||
|
||||
|
||||
def uses_cutedsl_ar_fusion() -> bool:
|
||||
"""Selected CuTe DSL owns both patterns, so the legacy workspace stands down."""
|
||||
return get_exec().comm.flashinfer_allreduce_fusion_backend == "cutedsl"
|
||||
|
||||
|
||||
def resolve_flashinfer_allreduce_fusion_backend() -> Optional[str]:
|
||||
"""The fusion backend for this process, or None when fusion is off.
|
||||
|
||||
@@ -713,7 +726,7 @@ def ensure_workspace_initialized(
|
||||
use_attn_tp_group: bool = True,
|
||||
):
|
||||
"""Ensure workspace is initialized."""
|
||||
if _flashinfer_allreduce_unavailable:
|
||||
if _flashinfer_allreduce_unavailable or uses_cutedsl_ar_fusion():
|
||||
return False
|
||||
|
||||
if not is_flashinfer_available() or _flashinfer_comm is None:
|
||||
|
||||
@@ -3,9 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from dataclasses import dataclass, replace
|
||||
from functools import lru_cache
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
@@ -20,8 +18,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _import_kernel_backend():
|
||||
# Imported here rather than at module scope: the CuTe DSL backend drags in
|
||||
# CUDA-only dependencies that CPU-side importers of this module never need.
|
||||
# Deferred: the backend drags in CUDA-only deps CPU importers never need.
|
||||
from flashinfer.comm import AllReduceFusionPattern, allreduce_fusion
|
||||
from flashinfer.comm.mnnvl_cutedsl import DEFAULT_CONFIG
|
||||
from flashinfer.comm.mnnvl_cutedsl_ar import MNNVLCuteDSLAllReduceFusionWorkspace
|
||||
@@ -34,6 +31,252 @@ def _import_kernel_backend():
|
||||
)
|
||||
|
||||
|
||||
# Mirrors the FlashInfer HT device kernel: warp size, bf16 elements per 16B vector.
|
||||
_WARP_SIZE = 32
|
||||
_VEC_BF16 = 8
|
||||
|
||||
# Kernel bound, with two non-consumer warps beside the reduction warps:
|
||||
# consumer_threads + (2 + reduction_warps) * WARP_SIZE <= _CUDA_BLOCK_THREADS.
|
||||
_CUDA_BLOCK_THREADS = 1024
|
||||
_HT_NON_REDUCTION_WARPS = 2
|
||||
_HT_REDUCTION_WARP_CHOICES = (1, 2, 4, 8)
|
||||
|
||||
_SUPPORTED_TP_SIZES = (2, 4, 8, 16)
|
||||
|
||||
|
||||
def _ht_shard_split(
|
||||
hidden_size: int, max_consumer_threads: int
|
||||
) -> tuple[int, int] | None:
|
||||
# The kernel shards a token into consumer_threads * 8 * vectors_per_thread
|
||||
# elements, and consumer_threads must divide its 16-byte vector count.
|
||||
packs = hidden_size // _VEC_BF16
|
||||
# packs // 2 keeps vectors_per_thread >= 2, as in the GB300 presets.
|
||||
limit = min(max_consumer_threads, packs // 2)
|
||||
for consumer_threads in range(
|
||||
limit - limit % _WARP_SIZE, _WARP_SIZE - 1, -_WARP_SIZE
|
||||
):
|
||||
if packs % consumer_threads == 0:
|
||||
return consumer_threads, packs // consumer_threads
|
||||
return None
|
||||
|
||||
|
||||
def _ht_reduction_warp_order(preferred: int) -> tuple[int, ...]:
|
||||
# Preset's value first, then nearest, fewer before more: each warp costs consumers.
|
||||
return tuple(
|
||||
sorted(
|
||||
_HT_REDUCTION_WARP_CHOICES,
|
||||
key=lambda warps: (warps > preferred, abs(warps - preferred)),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _ht_shard_major_is_legal(
|
||||
consumer_threads: int, rms_token_groups: int, tp_size: int
|
||||
) -> bool:
|
||||
# Shard-major RMS needs an integer number of reduction shards per RMS warp.
|
||||
rms_warps_per_token = (consumer_threads // rms_token_groups) // _WARP_SIZE
|
||||
return (
|
||||
rms_warps_per_token > 0
|
||||
and tp_size >= rms_warps_per_token
|
||||
and tp_size % rms_warps_per_token == 0
|
||||
)
|
||||
|
||||
|
||||
def _ht_retarget(preset, *, hidden_size: int, tp_size: int):
|
||||
"""``preset`` re-aimed at this shape, or None when no legal split exists.
|
||||
|
||||
Only shape-dependent fields move, except ``rms_shard_major``, which the
|
||||
kernel rejects unless tp is a multiple of the RMS warps per token.
|
||||
"""
|
||||
packs = hidden_size // _VEC_BF16
|
||||
if packs % tp_size:
|
||||
# Kernel: "hidden vector count must be divisible by tp".
|
||||
return None
|
||||
packs_per_shard = packs // tp_size
|
||||
for reduction_warps in _ht_reduction_warp_order(preset.reduction_warps):
|
||||
if packs_per_shard % (reduction_warps * _WARP_SIZE):
|
||||
continue
|
||||
split = _ht_shard_split(
|
||||
hidden_size,
|
||||
_CUDA_BLOCK_THREADS
|
||||
- (_HT_NON_REDUCTION_WARPS + reduction_warps) * _WARP_SIZE,
|
||||
)
|
||||
if split is None:
|
||||
continue
|
||||
consumer_threads, vectors_per_thread = split
|
||||
return replace(
|
||||
preset,
|
||||
consumer_threads=consumer_threads,
|
||||
vectors_per_thread=vectors_per_thread,
|
||||
reduction_warps=reduction_warps,
|
||||
rms_shard_major=preset.rms_shard_major
|
||||
and _ht_shard_major_is_legal(
|
||||
consumer_threads, preset.rms_token_groups, tp_size
|
||||
),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _routes(bounds, ll_target, bt_targets, ht_target):
|
||||
# Without HT the widest BT range takes the unbounded slot, so the profile
|
||||
# still covers the whole workspace capacity.
|
||||
from flashinfer.comm.mnnvl_cutedsl import MRangeDispatch
|
||||
|
||||
if ht_target is not None:
|
||||
return MRangeDispatch(
|
||||
upper_bounds=bounds,
|
||||
targets=(ll_target, *bt_targets, ht_target),
|
||||
)
|
||||
return MRangeDispatch(
|
||||
upper_bounds=(*bounds[: len(bt_targets)], None),
|
||||
targets=(ll_target, *bt_targets),
|
||||
)
|
||||
|
||||
|
||||
def _gb300_presets(*, wide_tp: bool):
|
||||
"""The shipped GB300 (bounds, LL, BT pair, HT) per operation, TP16 or TP8."""
|
||||
from flashinfer.comm.mnnvl_cutedsl.kernel_bt import (
|
||||
BT_ALL_REDUCE_GB300_TP8_H8192_PRESET_0,
|
||||
BT_ALL_REDUCE_GB300_TP8_H8192_PRESET_1,
|
||||
BT_ALL_REDUCE_GB300_TP16_H8192_PRESET_0,
|
||||
BT_ALL_REDUCE_GB300_TP16_H8192_PRESET_1,
|
||||
BT_FINALIZE_GB300_TP8_H8192_K10_PRESET_0,
|
||||
BT_FINALIZE_GB300_TP8_H8192_K10_PRESET_1,
|
||||
BT_FINALIZE_GB300_TP16_H8192_K10_PRESET_0,
|
||||
BT_FINALIZE_GB300_TP16_H8192_K10_PRESET_1,
|
||||
)
|
||||
from flashinfer.comm.mnnvl_cutedsl.kernel_ht import (
|
||||
HT_ALL_REDUCE_GB300_TP8_H8192,
|
||||
HT_ALL_REDUCE_GB300_TP16_H8192,
|
||||
HT_FINALIZE_GB300_TP8_H8192_K10,
|
||||
HT_FINALIZE_GB300_TP16_H8192_K10,
|
||||
)
|
||||
from flashinfer.comm.mnnvl_cutedsl.kernel_ll import (
|
||||
LL_ALL_REDUCE_GB300_TP8_H8192,
|
||||
LL_ALL_REDUCE_GB300_TP16_H8192,
|
||||
LL_FINALIZE_GB300_TP8_H8192_K10,
|
||||
LL_FINALIZE_GB300_TP16_H8192_K10,
|
||||
)
|
||||
|
||||
# FlashInfer's measured GB300 crossovers.
|
||||
if wide_tp:
|
||||
return {
|
||||
"finalize": (
|
||||
(7, 52, 703, None),
|
||||
LL_FINALIZE_GB300_TP16_H8192_K10,
|
||||
(
|
||||
BT_FINALIZE_GB300_TP16_H8192_K10_PRESET_0,
|
||||
BT_FINALIZE_GB300_TP16_H8192_K10_PRESET_1,
|
||||
),
|
||||
HT_FINALIZE_GB300_TP16_H8192_K10,
|
||||
),
|
||||
"all_reduce": (
|
||||
(5, 512, 959, None),
|
||||
LL_ALL_REDUCE_GB300_TP16_H8192,
|
||||
(
|
||||
BT_ALL_REDUCE_GB300_TP16_H8192_PRESET_0,
|
||||
BT_ALL_REDUCE_GB300_TP16_H8192_PRESET_1,
|
||||
),
|
||||
HT_ALL_REDUCE_GB300_TP16_H8192,
|
||||
),
|
||||
}
|
||||
return {
|
||||
"finalize": (
|
||||
(23, 48, 703, None),
|
||||
LL_FINALIZE_GB300_TP8_H8192_K10,
|
||||
(
|
||||
BT_FINALIZE_GB300_TP8_H8192_K10_PRESET_0,
|
||||
BT_FINALIZE_GB300_TP8_H8192_K10_PRESET_1,
|
||||
),
|
||||
HT_FINALIZE_GB300_TP8_H8192_K10,
|
||||
),
|
||||
"all_reduce": (
|
||||
(15, 256, 1024, None),
|
||||
LL_ALL_REDUCE_GB300_TP8_H8192,
|
||||
(
|
||||
BT_ALL_REDUCE_GB300_TP8_H8192_PRESET_0,
|
||||
BT_ALL_REDUCE_GB300_TP8_H8192_PRESET_1,
|
||||
),
|
||||
HT_ALL_REDUCE_GB300_TP8_H8192,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _retargeted_config(tp_size: int, hidden_size: int, top_k: int):
|
||||
"""A single-profile routing config for a shape FlashInfer does not ship.
|
||||
|
||||
Reuses the shipped GB300 crossovers, which were measured at tp=8/16
|
||||
hidden=8192 top_k=10 and are unmeasured elsewhere.
|
||||
"""
|
||||
from flashinfer.comm.mnnvl_cutedsl import (
|
||||
KernelTarget,
|
||||
MNNVLCuteDSLConfig,
|
||||
ProtocolKind,
|
||||
StaticProfile,
|
||||
)
|
||||
|
||||
shipped = _gb300_presets(wide_tp=tp_size >= 16)
|
||||
ht = {
|
||||
op: _ht_retarget(presets[3], hidden_size=hidden_size, tp_size=tp_size)
|
||||
for op, presets in shipped.items()
|
||||
}
|
||||
if any(preset is None for preset in ht.values()):
|
||||
# Both operations share the HT protocol state, so HT goes for both.
|
||||
ht = dict.fromkeys(ht)
|
||||
logger.warning(
|
||||
"MNNVL CuTe DSL: hidden_size=%d admits no HT shard split at "
|
||||
"tp_size=%d; serving this shape with the LL and BT routes only, "
|
||||
"which costs throughput at large token counts.",
|
||||
hidden_size,
|
||||
tp_size,
|
||||
)
|
||||
|
||||
def routes(op):
|
||||
bounds, ll, bt, _ = shipped[op]
|
||||
return _routes(
|
||||
bounds,
|
||||
KernelTarget(protocol=ProtocolKind.LL, preset=ll),
|
||||
tuple(KernelTarget(protocol=ProtocolKind.BT, preset=p) for p in bt),
|
||||
None
|
||||
if ht[op] is None
|
||||
else KernelTarget(protocol=ProtocolKind.HT, preset=ht[op]),
|
||||
)
|
||||
|
||||
profile = StaticProfile(
|
||||
tp_size=tp_size,
|
||||
hidden_size=hidden_size,
|
||||
top_k=top_k,
|
||||
dtype=torch.bfloat16,
|
||||
finalize_routes=routes("finalize"),
|
||||
all_reduce_routes=routes("all_reduce"),
|
||||
)
|
||||
return MNNVLCuteDSLConfig(profiles=(profile,))
|
||||
|
||||
|
||||
def _config_for_shape(default_config, *, tp_size: int, hidden_size: int, top_k: int):
|
||||
# MNNVLCuteDSLConfig.resolve matches (tp, hidden, top_k, dtype) exactly and
|
||||
# DEFAULT_CONFIG ships GB300 H=8192/K=10 only, so other shapes need one.
|
||||
for profile in default_config.profiles:
|
||||
if profile.matches(
|
||||
tp_size=tp_size,
|
||||
hidden_size=hidden_size,
|
||||
top_k=top_k,
|
||||
dtype=torch.bfloat16,
|
||||
):
|
||||
return default_config
|
||||
logger.info(
|
||||
"MNNVL CuTe DSL: no shipped profile for tp=%d hidden=%d top_k=%d; "
|
||||
"re-targeting the GB300 presets at this shape. Their M crossovers were "
|
||||
"measured at tp=8/16 hidden=8192 top_k=10, so benchmark against "
|
||||
"--flashinfer-allreduce-fusion-backend mnnvl before deploying a shape "
|
||||
"that matters.",
|
||||
tp_size,
|
||||
hidden_size,
|
||||
top_k,
|
||||
)
|
||||
return _retargeted_config(tp_size, hidden_size, top_k)
|
||||
|
||||
|
||||
def _with_early_finalize_shared_load(config):
|
||||
profiles = []
|
||||
updated_presets = 0
|
||||
@@ -64,17 +307,6 @@ def _with_early_finalize_shared_load(config):
|
||||
return replace(config, profiles=tuple(profiles))
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _WorkspaceSignature:
|
||||
hidden_size: int
|
||||
top_k: int
|
||||
rms_epsilon: float
|
||||
weight_bias: float
|
||||
max_m: int
|
||||
device_index: int
|
||||
process_group_identity: int
|
||||
|
||||
|
||||
class FlashInferMNNVLCuteDSLARFusion:
|
||||
"""One graph-stable workspace serving both supported fusion patterns."""
|
||||
|
||||
@@ -100,8 +332,14 @@ class FlashInferMNNVLCuteDSLARFusion:
|
||||
self.rms_epsilon = float(rms_epsilon)
|
||||
self.weight_bias = float(weight_bias)
|
||||
self.process_group = process_group
|
||||
# Cached: supports() runs per layer and must not re-enter c10d.
|
||||
self.tp_size = dist.get_world_size(process_group)
|
||||
if self.tp_size not in _SUPPORTED_TP_SIZES:
|
||||
raise ValueError(
|
||||
f"MNNVL CuTe DSL fusion supports tp_size in {_SUPPORTED_TP_SIZES}, "
|
||||
f"got {self.tp_size}"
|
||||
)
|
||||
self.device = torch.device(device)
|
||||
self._destroyed = False
|
||||
|
||||
with torch.cuda.device(self.device):
|
||||
self.device = torch.device("cuda", torch.cuda.current_device())
|
||||
@@ -129,11 +367,14 @@ class FlashInferMNNVLCuteDSLARFusion:
|
||||
self._patterns,
|
||||
default_config,
|
||||
) = _import_kernel_backend()
|
||||
# Only fused finalize launches have a completed shared-expert handoff;
|
||||
# standalone AllReduce kernels retain the safe load ordering.
|
||||
|
||||
shaped_config = _config_for_shape(
|
||||
default_config,
|
||||
tp_size=self.tp_size,
|
||||
hidden_size=self.hidden_size,
|
||||
top_k=self.top_k,
|
||||
)
|
||||
if get_spec().speculative_algorithm is None:
|
||||
self.workspace_config = _with_early_finalize_shared_load(default_config)
|
||||
self.workspace_config = _with_early_finalize_shared_load(shaped_config)
|
||||
else:
|
||||
# Early shared load is safe only for a single looping decode graph;
|
||||
# alternating draft/verify replays can read an unfinished buffer.
|
||||
@@ -142,9 +383,9 @@ class FlashInferMNNVLCuteDSLARFusion:
|
||||
"CuTe DSL finalize presets on the safe (non-early-load) "
|
||||
"ordering."
|
||||
)
|
||||
self.workspace_config = default_config
|
||||
self.workspace_config = shaped_config
|
||||
self.workspace = workspace_type(
|
||||
tp_size=dist.get_world_size(process_group),
|
||||
tp_size=self.tp_size,
|
||||
tp_rank=dist.get_rank(process_group),
|
||||
max_token_num=self.max_m,
|
||||
hidden_dim=self.hidden_size,
|
||||
@@ -166,10 +407,10 @@ class FlashInferMNNVLCuteDSLARFusion:
|
||||
dist.barrier(group=process_group)
|
||||
|
||||
def supports(self, m: int) -> bool:
|
||||
if self._destroyed or not 1 <= int(m) <= self.max_m:
|
||||
if not 1 <= int(m) <= self.max_m:
|
||||
return False
|
||||
return self.workspace.is_buffer_size_sufficient(
|
||||
tp_size=dist.get_world_size(self.process_group),
|
||||
tp_size=self.tp_size,
|
||||
num_tokens=int(m),
|
||||
hidden_dim=self.hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
@@ -184,27 +425,20 @@ class FlashInferMNNVLCuteDSLARFusion:
|
||||
gated_shared_output: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
gamma: torch.Tensor,
|
||||
norm_output: torch.Tensor | None = None,
|
||||
residual_output: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
m = int(permuted_indices.shape[0])
|
||||
if not self.supports(m):
|
||||
raise ValueError(f"workspace does not support M={m}")
|
||||
shape = (m, self.hidden_size)
|
||||
if norm_output is None:
|
||||
norm_output = torch.empty(shape, dtype=torch.bfloat16, device=self.device)
|
||||
if residual_output is None:
|
||||
residual_output = torch.empty(
|
||||
shape, dtype=torch.bfloat16, device=self.device
|
||||
)
|
||||
norm_output = torch.empty(shape, dtype=torch.bfloat16, device=self.device)
|
||||
residual_output = torch.empty(shape, dtype=torch.bfloat16, device=self.device)
|
||||
|
||||
pattern = self._patterns.kMoEFinalizeARResidualRMSNorm
|
||||
self._allreduce_fusion(
|
||||
input=routed_output,
|
||||
workspace=self.workspace,
|
||||
pattern=pattern,
|
||||
# The public API carries the caller's PDL intent. The backend's
|
||||
# routing profile owns the compiled choice and validates it.
|
||||
# Caller intent only; the routing profile owns the compiled choice.
|
||||
launch_with_pdl=True,
|
||||
residual_in=residual,
|
||||
residual_out=residual_output,
|
||||
@@ -224,16 +458,12 @@ class FlashInferMNNVLCuteDSLARFusion:
|
||||
local_contribution: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
gamma: torch.Tensor,
|
||||
norm_output: torch.Tensor | None = None,
|
||||
residual_output: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
m = int(local_contribution.shape[0])
|
||||
if not self.supports(m):
|
||||
raise ValueError(f"workspace does not support M={m}")
|
||||
if norm_output is None:
|
||||
norm_output = torch.empty_like(local_contribution)
|
||||
if residual_output is None:
|
||||
residual_output = torch.empty_like(local_contribution)
|
||||
norm_output = torch.empty_like(local_contribution)
|
||||
residual_output = torch.empty_like(local_contribution)
|
||||
|
||||
pattern = self._patterns.kARResidualRMSNorm
|
||||
self._allreduce_fusion(
|
||||
@@ -250,124 +480,43 @@ class FlashInferMNNVLCuteDSLARFusion:
|
||||
)
|
||||
return norm_output, residual_output
|
||||
|
||||
def destroy(self) -> None:
|
||||
if self._destroyed:
|
||||
return
|
||||
self.workspace.destroy()
|
||||
self._destroyed = True
|
||||
|
||||
|
||||
_WORKSPACES: dict[_WorkspaceSignature, FlashInferMNNVLCuteDSLARFusion] = {}
|
||||
_WORKSPACES_LOCK = threading.RLock()
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _max_workspace_instances() -> int:
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
value = int(envs.SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION_MAX_INSTANCES.get())
|
||||
if value < 1:
|
||||
raise ValueError(
|
||||
"SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION_MAX_INSTANCES must be positive"
|
||||
)
|
||||
return value
|
||||
_WORKSPACE: FlashInferMNNVLCuteDSLARFusion | None = None
|
||||
|
||||
|
||||
def get_flashinfer_mnnvl_cutedsl_ar_fusion(
|
||||
*,
|
||||
hidden_size: int | None = None,
|
||||
top_k: int | None = None,
|
||||
max_m: int | None = None,
|
||||
rms_epsilon: float | None = None,
|
||||
weight_bias: float | None = None,
|
||||
hidden_size: int,
|
||||
top_k: int,
|
||||
max_m: int,
|
||||
rms_epsilon: float,
|
||||
weight_bias: float,
|
||||
) -> FlashInferMNNVLCuteDSLARFusion:
|
||||
"""Lookup, or before graph capture create, the process-local workspace."""
|
||||
supplied = (hidden_size, top_k, max_m, rms_epsilon, weight_bias)
|
||||
if all(value is None for value in supplied):
|
||||
with _WORKSPACES_LOCK:
|
||||
if len(_WORKSPACES) == 1:
|
||||
return next(iter(_WORKSPACES.values()))
|
||||
if not _WORKSPACES:
|
||||
raise RuntimeError(
|
||||
"MNNVL CuTe DSL fusion workspace was not initialized before use"
|
||||
)
|
||||
raise RuntimeError(
|
||||
"multiple MNNVL CuTe DSL fusion workspaces exist; configuration "
|
||||
"arguments are required"
|
||||
)
|
||||
if any(value is None for value in supplied):
|
||||
raise TypeError(
|
||||
"hidden_size, top_k, max_m, rms_epsilon, and weight_bias must be "
|
||||
"supplied together"
|
||||
"""Build the process-local workspace. Must run before graph capture."""
|
||||
global _WORKSPACE
|
||||
if _WORKSPACE is not None:
|
||||
raise RuntimeError(
|
||||
"a second MNNVL CuTe DSL fusion workspace was requested; each one "
|
||||
"rendezvouses its own NVLS region, and a process serves one model"
|
||||
)
|
||||
if not torch.cuda.is_available():
|
||||
raise RuntimeError("MNNVL CuTe DSL fusion requires CUDA")
|
||||
|
||||
assert hidden_size is not None
|
||||
assert top_k is not None
|
||||
assert max_m is not None
|
||||
assert rms_epsilon is not None
|
||||
assert weight_bias is not None
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
device = torch.device("cuda", torch.cuda.current_device())
|
||||
process_group = get_parallel().tp_group.device_group
|
||||
domain = (
|
||||
int(hidden_size),
|
||||
int(top_k),
|
||||
float(rms_epsilon),
|
||||
float(weight_bias),
|
||||
int(device.index),
|
||||
id(process_group),
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
raise RuntimeError(
|
||||
"creating an MNNVL CuTe DSL fusion workspace during CUDA Graph "
|
||||
"capture is forbidden"
|
||||
)
|
||||
|
||||
_WORKSPACE = FlashInferMNNVLCuteDSLARFusion(
|
||||
hidden_size=hidden_size,
|
||||
top_k=top_k,
|
||||
max_m=max_m,
|
||||
rms_epsilon=rms_epsilon,
|
||||
weight_bias=weight_bias,
|
||||
process_group=get_parallel().tp_group.device_group,
|
||||
device=torch.device("cuda", torch.cuda.current_device()),
|
||||
)
|
||||
|
||||
with _WORKSPACES_LOCK:
|
||||
compatible = [
|
||||
(signature.max_m, instance)
|
||||
for signature, instance in _WORKSPACES.items()
|
||||
if (
|
||||
signature.hidden_size,
|
||||
signature.top_k,
|
||||
signature.rms_epsilon,
|
||||
signature.weight_bias,
|
||||
signature.device_index,
|
||||
signature.process_group_identity,
|
||||
)
|
||||
== domain
|
||||
and signature.max_m >= int(max_m)
|
||||
]
|
||||
if compatible:
|
||||
return min(compatible, key=lambda item: item[0])[1]
|
||||
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
raise RuntimeError(
|
||||
"creating an MNNVL CuTe DSL fusion workspace during CUDA Graph "
|
||||
"capture is forbidden"
|
||||
)
|
||||
if len(_WORKSPACES) >= _max_workspace_instances():
|
||||
raise RuntimeError(
|
||||
"MNNVL CuTe DSL fusion workspace instance limit exceeded; "
|
||||
"increase SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION_MAX_INSTANCES "
|
||||
"only when multiple model configurations intentionally coexist"
|
||||
)
|
||||
|
||||
signature = _WorkspaceSignature(
|
||||
hidden_size=int(hidden_size),
|
||||
top_k=int(top_k),
|
||||
rms_epsilon=float(rms_epsilon),
|
||||
weight_bias=float(weight_bias),
|
||||
max_m=int(max_m),
|
||||
device_index=int(device.index),
|
||||
process_group_identity=id(process_group),
|
||||
)
|
||||
instance = FlashInferMNNVLCuteDSLARFusion(
|
||||
hidden_size=hidden_size,
|
||||
top_k=top_k,
|
||||
max_m=max_m,
|
||||
rms_epsilon=rms_epsilon,
|
||||
weight_bias=weight_bias,
|
||||
process_group=process_group,
|
||||
device=device,
|
||||
)
|
||||
_WORKSPACES[signature] = instance
|
||||
return instance
|
||||
return _WORKSPACE
|
||||
|
||||
@@ -0,0 +1,478 @@
|
||||
"""FlashInfer MNNVL CuTe DSL AllReduce fusion, shared across architectures.
|
||||
|
||||
Two patterns share one workspace, both consumed at the next layer's input
|
||||
RMSNorm: AR + residual + RMSNorm, and the same with the MoE finalize and the
|
||||
shared-expert add folded in when the runner hands back a MoeFinalizeHandoff.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import logging
|
||||
from typing import Callable, Optional, Sequence
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.communicator import (
|
||||
CommunicateWithAllReduceAndLayerNormFn,
|
||||
LayerCommunicator,
|
||||
ScatterMode,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
|
||||
from sglang.srt.layers.moe import get_moe_a2a_backend
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.runtime_context import (
|
||||
cutedsl_moe_max_num_tokens,
|
||||
get_disagg,
|
||||
get_exec,
|
||||
get_flags,
|
||||
get_parallel,
|
||||
)
|
||||
|
||||
_LayerPredicate = Callable[[torch.nn.Module], bool]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _fused_norm_gamma(layernorm: torch.nn.Module) -> Optional[torch.Tensor]:
|
||||
"""The multiplier as applied -- GemmaRMSNorm's pre-folded w + 1. None declines."""
|
||||
if isinstance(layernorm, GemmaRMSNorm):
|
||||
return layernorm.gemma_weight
|
||||
if isinstance(layernorm, RMSNorm) and layernorm.has_weight:
|
||||
return layernorm.weight
|
||||
return None
|
||||
|
||||
|
||||
def _is_supported_forward_mode(forward_mode: ForwardMode) -> bool:
|
||||
return forward_mode in (
|
||||
ForwardMode.DECODE,
|
||||
ForwardMode.EXTEND,
|
||||
ForwardMode.TARGET_VERIFY,
|
||||
)
|
||||
|
||||
|
||||
def _resolve_max_m(*, max_running_requests: int | None) -> int:
|
||||
decode_config = get_exec().graph.cuda_graph_config.decode
|
||||
prefill_config = get_exec().graph.cuda_graph_config.prefill
|
||||
candidates = [
|
||||
cutedsl_moe_max_num_tokens(),
|
||||
max_running_requests,
|
||||
decode_config.max_bs,
|
||||
prefill_config.max_bs,
|
||||
*(decode_config.bs or []),
|
||||
*(prefill_config.bs or []),
|
||||
]
|
||||
positive = [
|
||||
int(value) for value in candidates if value is not None and int(value) > 0
|
||||
]
|
||||
if not positive:
|
||||
raise RuntimeError("framework reported no positive fusion workspace M bound")
|
||||
return max(positive)
|
||||
|
||||
|
||||
class MoeFinalizeHandoff(msgspec.Struct, frozen=True):
|
||||
"""Unfinalized routed output plus the separately gated shared contribution."""
|
||||
|
||||
routed_output: torch.Tensor
|
||||
expert_weights: torch.Tensor
|
||||
permuted_indices: torch.Tensor
|
||||
gated_shared_output: torch.Tensor
|
||||
m: int
|
||||
|
||||
@classmethod
|
||||
def from_flashinfer(
|
||||
cls,
|
||||
deferred_output,
|
||||
*,
|
||||
gated_shared_output: torch.Tensor,
|
||||
m: int,
|
||||
) -> MoeFinalizeHandoff:
|
||||
top_k = int(deferred_output.top_k)
|
||||
return cls(
|
||||
routed_output=deferred_output.gemm2_out.view(
|
||||
-1, deferred_output.gemm2_out.shape[-1]
|
||||
),
|
||||
expert_weights=deferred_output.expert_weights.view(-1, top_k)[:m],
|
||||
permuted_indices=deferred_output.expanded_idx_to_permuted_idx.view(
|
||||
-1, top_k
|
||||
)[:m],
|
||||
gated_shared_output=gated_shared_output,
|
||||
m=int(m),
|
||||
)
|
||||
|
||||
|
||||
class CuteDSLFusionService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
hidden_size: int,
|
||||
top_k: int,
|
||||
rms_epsilon: float,
|
||||
) -> None:
|
||||
self.hidden_size = int(hidden_size)
|
||||
self.top_k = int(top_k)
|
||||
self.rms_epsilon = float(rms_epsilon)
|
||||
self.max_m: int | None = None
|
||||
self._workspace = None
|
||||
|
||||
def prepare(self, *, max_m: int) -> None:
|
||||
if self._workspace is not None:
|
||||
assert self.max_m is not None
|
||||
if int(max_m) > self.max_m:
|
||||
raise RuntimeError(
|
||||
f"fusion workspace is already prepared for M_max={self.max_m}; "
|
||||
f"refusing M_max={max_m}"
|
||||
)
|
||||
return
|
||||
from sglang.srt.layers.flashinfer_mnnvl_cutedsl import (
|
||||
get_flashinfer_mnnvl_cutedsl_ar_fusion,
|
||||
)
|
||||
|
||||
workspace = get_flashinfer_mnnvl_cutedsl_ar_fusion(
|
||||
hidden_size=self.hidden_size,
|
||||
top_k=self.top_k,
|
||||
max_m=int(max_m),
|
||||
rms_epsilon=self.rms_epsilon,
|
||||
# _fused_norm_gamma() already returns the multiplier as applied.
|
||||
weight_bias=0.0,
|
||||
)
|
||||
self._workspace = workspace
|
||||
self.max_m = workspace.max_m
|
||||
|
||||
def supports(self, m: int) -> bool:
|
||||
"""False before prepare(), so it doubles as the readiness check."""
|
||||
return self._workspace is not None and self._workspace.supports(m)
|
||||
|
||||
def finalize(
|
||||
self,
|
||||
handoff: MoeFinalizeHandoff,
|
||||
residual: torch.Tensor,
|
||||
gamma: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert self._workspace is not None
|
||||
return self._workspace.moe_finalize_all_reduce_rms_norm(
|
||||
routed_output=handoff.routed_output,
|
||||
expert_weights=handoff.expert_weights,
|
||||
permuted_indices=handoff.permuted_indices,
|
||||
gated_shared_output=handoff.gated_shared_output,
|
||||
residual=residual,
|
||||
gamma=gamma,
|
||||
)
|
||||
|
||||
def all_reduce_residual_rms_norm(
|
||||
self,
|
||||
local_contribution: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
gamma: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert self._workspace is not None
|
||||
return self._workspace.all_reduce_residual_rms_norm(
|
||||
local_contribution=local_contribution,
|
||||
residual=residual,
|
||||
gamma=gamma,
|
||||
)
|
||||
|
||||
|
||||
class CuteDSLFusionLayerCommunicator(LayerCommunicator):
|
||||
fusion_service: CuteDSLFusionService | None = None
|
||||
|
||||
# The runner can defer and a successor or the final norm consumes the handoff.
|
||||
may_defer_moe_finalize: bool = False
|
||||
# False on the last layer: the final norm does not all-reduce.
|
||||
successor_absorbs_all_reduce: bool = False
|
||||
# A replicated output follows the reduction; moving it would scale that by tp.
|
||||
owes_local_reduction: bool = False
|
||||
|
||||
def prepare_attn(
|
||||
self,
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
quant_format: str = "",
|
||||
post_residual_addition=None,
|
||||
):
|
||||
if isinstance(hidden_states, MoeFinalizeHandoff):
|
||||
if not self._should_use_finalize(forward_batch, hidden_states.m):
|
||||
raise RuntimeError(
|
||||
"received deferred MoE output on an ineligible path "
|
||||
f"(M={hidden_states.m}, mode={forward_batch.forward_mode})"
|
||||
)
|
||||
if residual is None:
|
||||
raise RuntimeError("deferred MoE finalize requires residual input")
|
||||
gamma = _fused_norm_gamma(self.input_layernorm)
|
||||
if gamma is None:
|
||||
raise RuntimeError(
|
||||
"deferred MoE finalize requires a fusable RMSNorm flavour"
|
||||
)
|
||||
if post_residual_addition is not None:
|
||||
residual = residual + post_residual_addition
|
||||
assert self.fusion_service is not None
|
||||
hidden_states, residual = self.fusion_service.finalize(
|
||||
handoff=hidden_states, residual=residual, gamma=gamma
|
||||
)
|
||||
return self._finish_prepare_attn(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
if (
|
||||
residual is not None
|
||||
and hasattr(hidden_states, "_sglang_needs_allreduce_fusion")
|
||||
and hidden_states._sglang_needs_allreduce_fusion
|
||||
and self._can_consume_post_moe_all_reduce(
|
||||
forward_batch, int(hidden_states.shape[0])
|
||||
)
|
||||
):
|
||||
if post_residual_addition is not None:
|
||||
residual = residual + post_residual_addition
|
||||
assert self.fusion_service is not None
|
||||
hidden_states, residual = self.fusion_service.all_reduce_residual_rms_norm(
|
||||
local_contribution=hidden_states,
|
||||
residual=residual,
|
||||
gamma=_fused_norm_gamma(self.input_layernorm),
|
||||
)
|
||||
return self._finish_prepare_attn(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
return super().prepare_attn(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
quant_format=quant_format,
|
||||
post_residual_addition=post_residual_addition,
|
||||
)
|
||||
|
||||
def prepare_mlp(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
cache=None,
|
||||
):
|
||||
if cache is not None:
|
||||
self._context.cache = cache
|
||||
if self._should_use_all_reduce_rms_norm(
|
||||
forward_batch, int(hidden_states.shape[0]), residual
|
||||
):
|
||||
assert self.fusion_service is not None and residual is not None
|
||||
return self.fusion_service.all_reduce_residual_rms_norm(
|
||||
local_contribution=hidden_states,
|
||||
residual=residual,
|
||||
gamma=_fused_norm_gamma(self.post_attention_layernorm),
|
||||
)
|
||||
return super().prepare_mlp(hidden_states, residual, forward_batch, cache=cache)
|
||||
|
||||
def _should_use_all_reduce_rms_norm(
|
||||
self,
|
||||
forward_batch: ForwardBatch,
|
||||
m: int,
|
||||
residual: Optional[torch.Tensor],
|
||||
) -> bool:
|
||||
communicate_fn = self._communicate_with_all_reduce_and_layer_norm_fn
|
||||
if isinstance(communicate_fn, functools.partial):
|
||||
norm_fn = communicate_fn.func
|
||||
residual_input_mode = communicate_fn.keywords.get("residual_input_mode")
|
||||
else:
|
||||
norm_fn = communicate_fn
|
||||
residual_input_mode = None
|
||||
parallel = get_parallel()
|
||||
return (
|
||||
self._common_eligible(forward_batch, m)
|
||||
and residual is not None
|
||||
and _fused_norm_gamma(self.post_attention_layernorm) is not None
|
||||
and norm_fn
|
||||
is CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual
|
||||
and residual_input_mode is ScatterMode.TP_ATTN_FULL
|
||||
and self._context.attn_dp_size == 1
|
||||
and parallel.attn_tp_size == parallel.tp_size
|
||||
and not get_exec().comm.enable_quant_communications
|
||||
)
|
||||
|
||||
def _should_use_finalize(self, forward_batch: ForwardBatch, m: int) -> bool:
|
||||
return (
|
||||
self._common_eligible(forward_batch, m) and get_parallel().moe_ep_size == 1
|
||||
)
|
||||
|
||||
def _can_consume_post_moe_all_reduce(
|
||||
self, forward_batch: ForwardBatch, m: int
|
||||
) -> bool:
|
||||
"""Incoming, and independent of this layer's own successor."""
|
||||
return (
|
||||
self._common_eligible(forward_batch, m)
|
||||
and _fused_norm_gamma(self.input_layernorm) is not None
|
||||
and not get_exec().comm.enable_quant_communications
|
||||
)
|
||||
|
||||
def _can_absorb_post_moe_all_reduce(
|
||||
self, forward_batch: ForwardBatch, m: int
|
||||
) -> bool:
|
||||
"""Outgoing: skip our own all-reduce because the next layer absorbs it."""
|
||||
return (
|
||||
self.successor_absorbs_all_reduce
|
||||
and not self.owes_local_reduction
|
||||
and self._can_consume_post_moe_all_reduce(forward_batch, m)
|
||||
)
|
||||
|
||||
def should_defer_moe_finalize(
|
||||
self, forward_batch: ForwardBatch, m: int | None = None
|
||||
) -> bool:
|
||||
"""Deferring skips the post-experts all-reduce on the promise of a handoff."""
|
||||
if not self.may_defer_moe_finalize:
|
||||
return False
|
||||
if m is None:
|
||||
m = int(forward_batch.input_ids.shape[0])
|
||||
return self._should_use_finalize(forward_batch, m)
|
||||
|
||||
def _common_eligible(self, forward_batch: ForwardBatch, m: int) -> bool:
|
||||
parallel = get_parallel()
|
||||
return bool(
|
||||
self.fusion_service is not None
|
||||
and _is_supported_forward_mode(forward_batch.forward_mode)
|
||||
and self.fusion_service.supports(m)
|
||||
and not is_dp_attention_enabled()
|
||||
# Also forces moe_dp_size == 1, so a hybrid EP x MoE-TP reduction
|
||||
# always merges into the one TP reduction the workspace performs.
|
||||
and parallel.attn_cp_size == 1
|
||||
and not get_attn_tp_context().input_scattered
|
||||
and get_moe_a2a_backend().is_none()
|
||||
and self._context.tp_size > 1
|
||||
# Restates the base's moe-cp, MOE_FULL and SCATTERED refusals.
|
||||
and self.layer_scatter_modes.mlp_mode is ScatterMode.FULL
|
||||
)
|
||||
|
||||
def should_fuse_mlp_allreduce_with_next_layer(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> bool:
|
||||
m = int(forward_batch.input_ids.shape[0])
|
||||
if self.should_defer_moe_finalize(forward_batch, m):
|
||||
return True
|
||||
if self._can_absorb_post_moe_all_reduce(forward_batch, m):
|
||||
return True
|
||||
return super().should_fuse_mlp_allreduce_with_next_layer(forward_batch)
|
||||
|
||||
|
||||
def install_cutedsl_fusion(
|
||||
layers: Sequence[torch.nn.Module],
|
||||
*,
|
||||
hidden_size: int,
|
||||
top_k: int,
|
||||
rms_epsilon: float,
|
||||
can_defer_finalize: _LayerPredicate,
|
||||
requires_local_reduction: _LayerPredicate | None = None,
|
||||
final_norm_consumes_handoff: bool = False,
|
||||
label: str,
|
||||
) -> CuteDSLFusionService | None:
|
||||
"""One shared workspace handle per fusion-enabled layer, or None.
|
||||
|
||||
Every entry of ``layers`` must carry a ``layer_communicator``.
|
||||
"""
|
||||
if get_flags().moe.in_speculative_scope:
|
||||
# A draft shares the target's process, which holds one workspace.
|
||||
return None
|
||||
|
||||
fusion_layers = [
|
||||
layer
|
||||
for layer in layers
|
||||
if isinstance(layer.layer_communicator, CuteDSLFusionLayerCommunicator)
|
||||
]
|
||||
if not fusion_layers:
|
||||
return None
|
||||
|
||||
# The workspace compiles one epsilon and cannot see a per-layer one.
|
||||
for layer in fusion_layers:
|
||||
for norm in (
|
||||
layer.layer_communicator.input_layernorm,
|
||||
layer.layer_communicator.post_attention_layernorm,
|
||||
):
|
||||
if _fused_norm_gamma(norm) is None:
|
||||
continue
|
||||
if float(norm.variance_epsilon) != float(rms_epsilon):
|
||||
raise RuntimeError(
|
||||
f"{label} CuTe DSL fusion compiles one workspace for "
|
||||
f"rms_epsilon={rms_epsilon}, but a fused norm uses "
|
||||
f"{norm.variance_epsilon}"
|
||||
)
|
||||
|
||||
service = CuteDSLFusionService(
|
||||
hidden_size=hidden_size,
|
||||
top_k=top_k,
|
||||
rms_epsilon=rms_epsilon,
|
||||
)
|
||||
for index, layer in enumerate(layers):
|
||||
communicator = layer.layer_communicator
|
||||
if not isinstance(communicator, CuteDSLFusionLayerCommunicator):
|
||||
continue
|
||||
successor = layers[index + 1] if index + 1 < len(layers) else None
|
||||
if successor is None:
|
||||
has_consumer = final_norm_consumes_handoff
|
||||
else:
|
||||
has_consumer = isinstance(
|
||||
successor.layer_communicator, CuteDSLFusionLayerCommunicator
|
||||
)
|
||||
communicator.fusion_service = service
|
||||
communicator.may_defer_moe_finalize = (
|
||||
bool(can_defer_finalize(layer)) and has_consumer
|
||||
)
|
||||
communicator.successor_absorbs_all_reduce = successor is not None and (
|
||||
isinstance(successor.layer_communicator, CuteDSLFusionLayerCommunicator)
|
||||
)
|
||||
communicator.owes_local_reduction = (
|
||||
requires_local_reduction is not None and requires_local_reduction(layer)
|
||||
)
|
||||
logger.info(
|
||||
"Installed one %s FlashInfer MNNVL CuTe DSL fusion handle for %d of %d layers "
|
||||
"(%d can defer the MoE finalize)",
|
||||
label,
|
||||
len(fusion_layers),
|
||||
len(layers),
|
||||
sum(layer.layer_communicator.may_defer_moe_finalize for layer in fusion_layers),
|
||||
)
|
||||
return service
|
||||
|
||||
|
||||
def prepare_cutedsl_fusion(
|
||||
model: torch.nn.Module, *, max_running_requests: int | None
|
||||
) -> None:
|
||||
"""Build the workspace of every service installed anywhere in ``model``.
|
||||
|
||||
Scans the module tree, so a wrapper holding the model as a submodule (a VLM's
|
||||
language_model) needs no hook of its own.
|
||||
"""
|
||||
communicators = [
|
||||
communicator
|
||||
for module in model.modules()
|
||||
# Most modules carry no layer_communicator.
|
||||
if isinstance(
|
||||
communicator := module.__dict__.get("layer_communicator"),
|
||||
CuteDSLFusionLayerCommunicator,
|
||||
)
|
||||
]
|
||||
if not communicators or any(c.fusion_service is None for c in communicators):
|
||||
raise ValueError(
|
||||
"--flashinfer-allreduce-fusion-backend cutedsl is set, but "
|
||||
f"{type(model).__name__} installed no CuTe DSL fusion service, so no "
|
||||
"allreduce fusion would run at all. Drop the flag, or choose 'auto', "
|
||||
"'trtllm' or 'mnnvl'."
|
||||
)
|
||||
if get_disagg().enable_pdmux:
|
||||
raise RuntimeError(
|
||||
"FlashInfer MNNVL CuTe DSL fusion does not support concurrent PDMux "
|
||||
"streams sharing one mutable workspace"
|
||||
)
|
||||
services = {id(c.fusion_service): c.fusion_service for c in communicators}
|
||||
max_m = _resolve_max_m(max_running_requests=max_running_requests)
|
||||
for service in services.values():
|
||||
service.prepare(max_m=max_m)
|
||||
logger.info(
|
||||
"Prepared the FlashInfer MNNVL CuTe DSL fusion workspace for M_max=%d "
|
||||
"(%d fused layers)",
|
||||
max_m,
|
||||
len(communicators),
|
||||
)
|
||||
@@ -1495,13 +1495,7 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
|
||||
|
||||
result = trtllm_fp4_block_scale_moe(**moe_kwargs)
|
||||
if defer_finalize:
|
||||
gemm2_out, expert_weights, expanded_idx_to_permuted_idx = result[:3]
|
||||
result = FlashInferTrtllmDeferredFinalizeOutput(
|
||||
gemm2_out=gemm2_out,
|
||||
expert_weights=expert_weights,
|
||||
expanded_idx_to_permuted_idx=expanded_idx_to_permuted_idx,
|
||||
top_k=topk_config.top_k,
|
||||
)
|
||||
result = _make_deferred_finalize_output(result, top_k=topk_config.top_k)
|
||||
else:
|
||||
result = result[0]
|
||||
|
||||
|
||||
@@ -1,350 +0,0 @@
|
||||
"""Qwen3.5 integration for FlashInfer MNNVL CuTe DSL AllReduce fusion."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.arg_groups.overrides import cutedsl_moe_max_num_tokens
|
||||
from sglang.srt.layers.communicator import (
|
||||
CommunicateWithAllReduceAndLayerNormFn,
|
||||
LayerCommunicator,
|
||||
ScatterMode,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.moe import get_moe_a2a_backend
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.runtime_context import (
|
||||
get_disagg,
|
||||
get_exec,
|
||||
get_parallel,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def is_supported_forward_mode(forward_mode: ForwardMode) -> bool:
|
||||
return forward_mode in (
|
||||
ForwardMode.DECODE,
|
||||
ForwardMode.EXTEND,
|
||||
ForwardMode.TARGET_VERIFY,
|
||||
)
|
||||
|
||||
|
||||
def resolve_max_m(model_runner) -> int:
|
||||
"""Use framework token bounds as the workspace-capacity source of truth."""
|
||||
decode_config = get_exec().graph.cuda_graph_config.decode
|
||||
prefill_config = get_exec().graph.cuda_graph_config.prefill
|
||||
candidates = [
|
||||
cutedsl_moe_max_num_tokens(model_runner.server_args),
|
||||
model_runner.max_running_requests,
|
||||
decode_config.max_bs,
|
||||
prefill_config.max_bs,
|
||||
*(decode_config.bs or []),
|
||||
*(prefill_config.bs or []),
|
||||
]
|
||||
positive = [
|
||||
int(value) for value in candidates if value is not None and int(value) > 0
|
||||
]
|
||||
if not positive:
|
||||
raise RuntimeError("framework reported no positive fusion workspace M bound")
|
||||
return max(positive)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Qwen35MoeFinalizeHandoff:
|
||||
"""Unfinalized routed output plus the separately gated shared contribution."""
|
||||
|
||||
routed_output: torch.Tensor
|
||||
expert_weights: torch.Tensor
|
||||
permuted_indices: torch.Tensor
|
||||
gated_shared_output: torch.Tensor
|
||||
m: int
|
||||
|
||||
@classmethod
|
||||
def from_flashinfer(
|
||||
cls,
|
||||
deferred_output,
|
||||
*,
|
||||
gated_shared_output: torch.Tensor,
|
||||
m: int,
|
||||
) -> Qwen35MoeFinalizeHandoff:
|
||||
top_k = int(deferred_output.top_k)
|
||||
return cls(
|
||||
routed_output=deferred_output.gemm2_out.view(
|
||||
-1, deferred_output.gemm2_out.shape[-1]
|
||||
),
|
||||
expert_weights=deferred_output.expert_weights.view(-1, top_k)[:m],
|
||||
permuted_indices=deferred_output.expanded_idx_to_permuted_idx.view(
|
||||
-1, top_k
|
||||
)[:m],
|
||||
gated_shared_output=gated_shared_output,
|
||||
m=int(m),
|
||||
)
|
||||
|
||||
|
||||
class Qwen35FlashInferFusionService:
|
||||
"""A lightweight model handle for the process-local FlashInfer workspace."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
hidden_size: int,
|
||||
top_k: int,
|
||||
rms_epsilon: float,
|
||||
) -> None:
|
||||
self.hidden_size = int(hidden_size)
|
||||
self.top_k = int(top_k)
|
||||
self.rms_epsilon = float(rms_epsilon)
|
||||
self.max_m: int | None = None
|
||||
self._workspace = None
|
||||
|
||||
@property
|
||||
def is_prepared(self) -> bool:
|
||||
return self._workspace is not None
|
||||
|
||||
def prepare(self, *, max_m: int) -> None:
|
||||
if self._workspace is not None:
|
||||
assert self.max_m is not None
|
||||
if int(max_m) > self.max_m:
|
||||
raise RuntimeError(
|
||||
f"fusion workspace is already prepared for M_max={self.max_m}; "
|
||||
f"refusing M_max={max_m}"
|
||||
)
|
||||
return
|
||||
from sglang.srt.layers.flashinfer_mnnvl_cutedsl import (
|
||||
get_flashinfer_mnnvl_cutedsl_ar_fusion,
|
||||
)
|
||||
|
||||
workspace = get_flashinfer_mnnvl_cutedsl_ar_fusion(
|
||||
hidden_size=self.hidden_size,
|
||||
top_k=self.top_k,
|
||||
max_m=int(max_m),
|
||||
rms_epsilon=self.rms_epsilon,
|
||||
# GemmaRMSNorm.gemma_weight is already checkpoint weight + 1.
|
||||
weight_bias=0.0,
|
||||
)
|
||||
self._workspace = workspace
|
||||
self.max_m = workspace.max_m
|
||||
|
||||
def supports(self, m: int) -> bool:
|
||||
if self._workspace is None or self.max_m is None:
|
||||
return False
|
||||
return 1 <= int(m) <= self.max_m and self._workspace.supports(m)
|
||||
|
||||
def finalize(
|
||||
self,
|
||||
handoff: Qwen35MoeFinalizeHandoff,
|
||||
residual: torch.Tensor,
|
||||
gamma: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
self._validate_finalize(handoff, residual, gamma)
|
||||
assert self._workspace is not None
|
||||
return self._workspace.moe_finalize_all_reduce_rms_norm(
|
||||
routed_output=handoff.routed_output,
|
||||
expert_weights=handoff.expert_weights,
|
||||
permuted_indices=handoff.permuted_indices,
|
||||
gated_shared_output=handoff.gated_shared_output,
|
||||
residual=residual,
|
||||
gamma=gamma,
|
||||
)
|
||||
|
||||
def all_reduce_residual_rms_norm(
|
||||
self,
|
||||
local_contribution: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
gamma: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
self._validate_matrix(local_contribution, "local_contribution")
|
||||
self._validate_matrix(residual, "residual", m=local_contribution.shape[0])
|
||||
self._validate_gamma(gamma)
|
||||
if not self.supports(local_contribution.shape[0]):
|
||||
raise ValueError(f"unsupported M={local_contribution.shape[0]}")
|
||||
assert self._workspace is not None
|
||||
return self._workspace.all_reduce_residual_rms_norm(
|
||||
local_contribution=local_contribution,
|
||||
residual=residual,
|
||||
gamma=gamma,
|
||||
)
|
||||
|
||||
def _validate_finalize(
|
||||
self,
|
||||
handoff: Qwen35MoeFinalizeHandoff,
|
||||
residual: torch.Tensor,
|
||||
gamma: torch.Tensor,
|
||||
) -> None:
|
||||
if not self.supports(handoff.m):
|
||||
raise ValueError(f"unsupported M={handoff.m}")
|
||||
self._validate_matrix(handoff.routed_output, "routed_output", exact_m=False)
|
||||
expected_metadata = (handoff.m, self.top_k)
|
||||
if tuple(handoff.expert_weights.shape) != expected_metadata:
|
||||
raise ValueError("expert_weights must have shape [M, top_k]")
|
||||
if tuple(handoff.permuted_indices.shape) != expected_metadata:
|
||||
raise ValueError("permuted_indices must have shape [M, top_k]")
|
||||
if handoff.expert_weights.dtype != torch.bfloat16:
|
||||
raise ValueError("expert_weights must be BF16")
|
||||
if handoff.permuted_indices.dtype != torch.int32:
|
||||
raise ValueError("permuted_indices must be Int32")
|
||||
self._validate_matrix(
|
||||
handoff.gated_shared_output, "gated_shared_output", m=handoff.m
|
||||
)
|
||||
self._validate_matrix(residual, "residual", m=handoff.m)
|
||||
self._validate_gamma(gamma)
|
||||
|
||||
def _validate_matrix(
|
||||
self,
|
||||
tensor: torch.Tensor,
|
||||
name: str,
|
||||
*,
|
||||
m: int | None = None,
|
||||
exact_m: bool = True,
|
||||
) -> None:
|
||||
if tensor.ndim != 2 or tensor.shape[1] != self.hidden_size:
|
||||
raise ValueError(f"{name} must have shape [M, hidden_size]")
|
||||
if m is not None and exact_m and tensor.shape[0] != int(m):
|
||||
raise ValueError(f"{name} has the wrong M dimension")
|
||||
if tensor.dtype != torch.bfloat16 or not tensor.is_contiguous():
|
||||
raise ValueError(f"{name} must be contiguous BF16")
|
||||
|
||||
def _validate_gamma(self, gamma: torch.Tensor) -> None:
|
||||
if (
|
||||
tuple(gamma.shape) != (self.hidden_size,)
|
||||
or gamma.dtype != torch.bfloat16
|
||||
or not gamma.is_contiguous()
|
||||
):
|
||||
raise ValueError("gamma must be contiguous BF16 [hidden_size]")
|
||||
|
||||
|
||||
class Qwen35FlashInferLayerCommunicator(LayerCommunicator):
|
||||
"""Qwen-only hooks; generic LayerCommunicator remains backend agnostic."""
|
||||
|
||||
fusion_service: Qwen35FlashInferFusionService | None = None
|
||||
|
||||
def prepare_attn(
|
||||
self,
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
quant_format: str = "",
|
||||
post_residual_addition=None,
|
||||
):
|
||||
if isinstance(hidden_states, Qwen35MoeFinalizeHandoff):
|
||||
if not self.should_use_finalize(forward_batch, hidden_states.m):
|
||||
raise RuntimeError("received deferred MoE output on an ineligible path")
|
||||
if residual is None:
|
||||
raise RuntimeError("deferred MoE finalize requires residual input")
|
||||
if not hasattr(self.input_layernorm, "gemma_weight"):
|
||||
raise RuntimeError("deferred Qwen finalize requires GemmaRMSNorm")
|
||||
if post_residual_addition is not None:
|
||||
residual = residual + post_residual_addition
|
||||
assert self.fusion_service is not None
|
||||
return self.fusion_service.finalize(
|
||||
hidden_states, residual, self.input_layernorm.gemma_weight
|
||||
)
|
||||
return super().prepare_attn(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
quant_format=quant_format,
|
||||
post_residual_addition=post_residual_addition,
|
||||
)
|
||||
|
||||
def prepare_mlp(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
cache=None,
|
||||
):
|
||||
if cache is not None:
|
||||
self._context.cache = cache
|
||||
if self.should_use_all_reduce_rms_norm(
|
||||
forward_batch, int(hidden_states.shape[0]), residual
|
||||
):
|
||||
assert self.fusion_service is not None and residual is not None
|
||||
return self.fusion_service.all_reduce_residual_rms_norm(
|
||||
hidden_states,
|
||||
residual,
|
||||
self.post_attention_layernorm.gemma_weight,
|
||||
)
|
||||
return super().prepare_mlp(hidden_states, residual, forward_batch, cache=cache)
|
||||
|
||||
def should_use_all_reduce_rms_norm(
|
||||
self,
|
||||
forward_batch: ForwardBatch,
|
||||
m: int,
|
||||
residual: Optional[torch.Tensor],
|
||||
) -> bool:
|
||||
communicate_fn = self._communicate_with_all_reduce_and_layer_norm_fn
|
||||
norm_fn = getattr(communicate_fn, "func", communicate_fn)
|
||||
residual_input_mode = getattr(communicate_fn, "keywords", {}).get(
|
||||
"residual_input_mode"
|
||||
)
|
||||
parallel = get_parallel()
|
||||
return (
|
||||
self._common_eligible(forward_batch, m)
|
||||
and residual is not None
|
||||
and hasattr(self.post_attention_layernorm, "gemma_weight")
|
||||
and norm_fn
|
||||
is CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual
|
||||
and residual_input_mode is ScatterMode.TP_ATTN_FULL
|
||||
and self._context.attn_dp_size == 1
|
||||
and parallel.attn_tp_size == parallel.tp_size
|
||||
and not get_exec().comm.enable_quant_communications
|
||||
)
|
||||
|
||||
def should_use_finalize(self, forward_batch: ForwardBatch, m: int) -> bool:
|
||||
parallel = get_parallel()
|
||||
return (
|
||||
self._common_eligible(forward_batch, m)
|
||||
and self.layer_scatter_modes.mlp_mode is not ScatterMode.SCATTERED
|
||||
and parallel.moe_ep_size == 1
|
||||
)
|
||||
|
||||
def _common_eligible(self, forward_batch: ForwardBatch, m: int) -> bool:
|
||||
parallel = get_parallel()
|
||||
return bool(
|
||||
self.fusion_service is not None
|
||||
and self.fusion_service.is_prepared
|
||||
and is_supported_forward_mode(forward_batch.forward_mode)
|
||||
and self.fusion_service.supports(m)
|
||||
and not is_dp_attention_enabled()
|
||||
and parallel.attn_cp_size == 1
|
||||
and not get_attn_tp_context().input_scattered
|
||||
and get_moe_a2a_backend().is_none()
|
||||
and self._context.tp_size > 1
|
||||
)
|
||||
|
||||
def should_fuse_mlp_allreduce_with_next_layer(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> bool:
|
||||
m = (
|
||||
int(forward_batch.input_ids.shape[0])
|
||||
if getattr(forward_batch, "input_ids", None) is not None
|
||||
else 0
|
||||
)
|
||||
if self.should_use_finalize(forward_batch, m):
|
||||
# The Qwen model consumes the final layer's handoff with its final
|
||||
# GemmaRMSNorm, so this is intentionally also true for that layer.
|
||||
return True
|
||||
return super().should_fuse_mlp_allreduce_with_next_layer(forward_batch)
|
||||
|
||||
|
||||
def prepare_qwen35_flashinfer_fusion(model, model_runner) -> None:
|
||||
service = getattr(model, "flashinfer_mnnvl_cutedsl_fusion", None)
|
||||
if service is None:
|
||||
return
|
||||
if get_disagg().enable_pdmux:
|
||||
raise RuntimeError(
|
||||
"FlashInfer MNNVL CuTe DSL fusion does not support concurrent PDMux "
|
||||
"streams sharing one mutable workspace"
|
||||
)
|
||||
service.prepare(max_m=resolve_max_m(model_runner))
|
||||
logger.info(
|
||||
"Prepared Qwen3.5 FlashInfer MNNVL CuTe DSL fusion workspace for M_max=%d",
|
||||
service.max_m,
|
||||
)
|
||||
@@ -288,11 +288,29 @@ class BaseRunner(ABC):
|
||||
with custom_all_reduce.register_graph_buffers).
|
||||
"""
|
||||
mr = self.model_runner
|
||||
from sglang.srt.layers.communicator import FUSE_ALLREDUCE_MAX_BATCH_SIZE
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import (
|
||||
pre_initialize_workspaces,
|
||||
resolve_flashinfer_allreduce_fusion_backend,
|
||||
uses_cutedsl_ar_fusion,
|
||||
)
|
||||
|
||||
if get_exec().comm.flashinfer_allreduce_fusion_backend is None:
|
||||
return
|
||||
|
||||
from sglang.srt.layers.communicator import FUSE_ALLREDUCE_MAX_BATCH_SIZE
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import pre_initialize_workspaces
|
||||
if uses_cutedsl_ar_fusion():
|
||||
# Nothing else resolves the configured backend, so check the platform.
|
||||
resolve_flashinfer_allreduce_fusion_backend()
|
||||
if not mr.is_draft_worker:
|
||||
# A draft installs no fusion communicator.
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import (
|
||||
prepare_cutedsl_fusion,
|
||||
)
|
||||
|
||||
prepare_cutedsl_fusion(
|
||||
mr.model, max_running_requests=mr.max_running_requests
|
||||
)
|
||||
return
|
||||
|
||||
pre_initialize_workspaces(
|
||||
max_token_num=FUSE_ALLREDUCE_MAX_BATCH_SIZE,
|
||||
|
||||
@@ -81,6 +81,7 @@ from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
|
||||
from sglang.srt.layers.dcp.planner import (
|
||||
prepare_decode_context_parallel_metadata,
|
||||
)
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -575,6 +576,9 @@ class DeepseekV2MoE(nn.Module):
|
||||
self.moe_ep_size = get_parallel().moe_ep_size
|
||||
self.routed_scaling_factor = config.routed_scaling_factor
|
||||
self.n_shared_experts = config.n_shared_experts
|
||||
self._deferred_finalize_max_tokens = (
|
||||
envs.SGLANG_MOE_DEFERRED_FINALIZE_MAX_TOKENS.get()
|
||||
)
|
||||
|
||||
n_shared_experts = (
|
||||
0 if config.n_shared_experts is None else int(config.n_shared_experts)
|
||||
@@ -1061,6 +1065,10 @@ class DeepseekV2MoE(nn.Module):
|
||||
and not self._shared_expert_tp1
|
||||
and topk_output.format == TopKOutputFormat.BYPASSED
|
||||
and self.experts.supports_deferred_finalize
|
||||
and (
|
||||
self._deferred_finalize_max_tokens <= 0
|
||||
or hidden_states.shape[0] <= self._deferred_finalize_max_tokens
|
||||
)
|
||||
)
|
||||
if deferred_finalize:
|
||||
final_hidden_states = self.experts.forward_deferred_finalize(
|
||||
@@ -1094,6 +1102,17 @@ class DeepseekV2MoE(nn.Module):
|
||||
# The routed-input pre-quant was already joined inside the routed MoE apply.
|
||||
current_stream.wait_stream(self.alt_stream)
|
||||
|
||||
if deferred_finalize and get_forward().defer_moe_finalize:
|
||||
# deferred_finalize excludes _shared_expert_tp1, so the shared add folds in.
|
||||
assert shared_output is not None
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import MoeFinalizeHandoff
|
||||
|
||||
return MoeFinalizeHandoff.from_flashinfer(
|
||||
final_hidden_states,
|
||||
gated_shared_output=shared_output,
|
||||
m=hidden_states.shape[0],
|
||||
)
|
||||
|
||||
all_reduce_done = False
|
||||
if deferred_finalize:
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
|
||||
@@ -2482,6 +2501,10 @@ class DeepseekV2AttentionMLA(
|
||||
return quant_config
|
||||
|
||||
|
||||
def _use_mnnvl_cutedsl_fusion() -> bool:
|
||||
return _is_cuda and uses_cutedsl_ar_fusion()
|
||||
|
||||
|
||||
class DeepseekV2DecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -2586,11 +2609,17 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
|
||||
self._gfx95_quant_format = self._detect_gfx95_quant_format()
|
||||
|
||||
communicator_cls = (
|
||||
DSACPLayerCommunicator
|
||||
if get_parallel().enable_prefill_cp
|
||||
else LayerCommunicator
|
||||
)
|
||||
if get_parallel().enable_prefill_cp:
|
||||
communicator_cls = DSACPLayerCommunicator
|
||||
elif not is_nextn and _use_mnnvl_cutedsl_fusion():
|
||||
# Dense layers too: selecting cutedsl turns the legacy fusion off.
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import (
|
||||
CuteDSLFusionLayerCommunicator,
|
||||
)
|
||||
|
||||
communicator_cls = CuteDSLFusionLayerCommunicator
|
||||
else:
|
||||
communicator_cls = LayerCommunicator
|
||||
self.layer_communicator = communicator_cls(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
input_layernorm=self.input_layernorm,
|
||||
@@ -2696,6 +2725,8 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
isinstance(self.mlp, DeepseekV2MoE)
|
||||
and not self.mlp.experts.moe_runner_config.inplace
|
||||
and not torch.compiler.is_compiling()
|
||||
# A deferred MoE finalize handoff from the previous layer is no buffer.
|
||||
and isinstance(hidden_states_orig, torch.Tensor)
|
||||
):
|
||||
from sglang.srt.layers.moe.moe_runner.base import moe_output_buffer_ctx
|
||||
|
||||
@@ -2917,6 +2948,34 @@ class DeepseekV2Model(nn.Module):
|
||||
# llama_4_scaling: for supporting Mistral-Large-3 model
|
||||
self.llama_4_scaling_config = getattr(config, "llama_4_scaling", None)
|
||||
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import install_cutedsl_fusion
|
||||
|
||||
install_cutedsl_fusion(
|
||||
# PP pads self.layers with PPMissingLayer, which has no communicator.
|
||||
# A stage's last layer has no successor, so it keeps its reduction.
|
||||
self.layers[self.start_layer : self.end_layer],
|
||||
hidden_size=config.hidden_size,
|
||||
top_k=config.num_experts_per_tok,
|
||||
rms_epsilon=config.rms_norm_eps,
|
||||
# Only the dual-stream path hands off, which needs an unfused shared
|
||||
# expert; with V3/R1's default shared-experts fusion only AR + norm fuse.
|
||||
# A TP1 shared expert is added after the all-reduce and cannot move it.
|
||||
can_defer_finalize=lambda layer: (
|
||||
isinstance(layer.mlp, DeepseekV2MoE)
|
||||
and layer.mlp.experts.supports_deferred_finalize
|
||||
and layer.mlp.num_fused_shared_experts == 0
|
||||
and not layer.mlp._shared_expert_tp1
|
||||
),
|
||||
# A TP1-replicated output must not ride into the next layer's reduction.
|
||||
requires_local_reduction=lambda layer: (
|
||||
layer.mlp._shared_expert_tp1
|
||||
if isinstance(layer.mlp, DeepseekV2MoE)
|
||||
else layer.mlp.tp_size == 1
|
||||
),
|
||||
final_norm_consumes_handoff=False,
|
||||
label="DeepSeek-V3/GLM",
|
||||
)
|
||||
|
||||
def get_input_embeddings(self) -> torch.Tensor:
|
||||
return self.embed_tokens
|
||||
|
||||
@@ -3385,6 +3444,8 @@ def dsv2_flashinfer_moe_dual_stream_graph(
|
||||
fuse_mlp_allreduce=fuse_mlp_allreduce,
|
||||
mlp_reduce_scatter=mlp_reduce_scatter,
|
||||
flashinfer_trtllm_bypass=True,
|
||||
# The op's Tensor schema cannot carry a MoeFinalizeHandoff.
|
||||
defer_moe_finalize=False,
|
||||
):
|
||||
return moe_fusion.forward_normal_dual_stream(hidden_states)
|
||||
|
||||
|
||||
@@ -51,6 +51,7 @@ from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -337,8 +338,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
num_fused_shared_experts=self.num_fused_shared_experts,
|
||||
inplace=not _needs_hidden_after_experts,
|
||||
enable_qwen35_fp8_deferred_finalize=(
|
||||
config.model_type == "qwen3_5_moe_text"
|
||||
and envs.SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION.get()
|
||||
config.model_type == "qwen3_5_moe_text" and uses_cutedsl_ar_fusion()
|
||||
),
|
||||
)
|
||||
|
||||
@@ -865,11 +865,9 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
if defer_finalize:
|
||||
if shared_output is None:
|
||||
raise RuntimeError("Qwen deferred finalize requires shared output")
|
||||
from sglang.srt.layers.moe.qwen35_flashinfer_fusion import (
|
||||
Qwen35MoeFinalizeHandoff,
|
||||
)
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import MoeFinalizeHandoff
|
||||
|
||||
return Qwen35MoeFinalizeHandoff.from_flashinfer(
|
||||
return MoeFinalizeHandoff.from_flashinfer(
|
||||
final_hidden_states,
|
||||
gated_shared_output=shared_output,
|
||||
m=num_tokens,
|
||||
|
||||
@@ -47,6 +47,7 @@ from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
|
||||
|
||||
# Layers - Others
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm
|
||||
@@ -172,10 +173,7 @@ def _disable_shared_experts_fusion() -> bool:
|
||||
# intent through the accessor's fallback.
|
||||
# The deferred-finalize ABI needs the shared expert as a separate, gated
|
||||
# local contribution; it cannot consume a shared slot fused into routed MoE.
|
||||
return bool(
|
||||
envs.SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION.get()
|
||||
or is_shared_experts_fusion_disabled()
|
||||
)
|
||||
return bool(uses_cutedsl_ar_fusion() or is_shared_experts_fusion_disabled())
|
||||
|
||||
|
||||
def _maybe_enable_silu_fp4_quant_fusion(mlp: nn.Module) -> None:
|
||||
@@ -209,17 +207,17 @@ def _use_mnnvl_cutedsl_fusion(config: Qwen3_5TextConfig, is_nextn: bool) -> bool
|
||||
return bool(
|
||||
not is_nextn
|
||||
and config.model_type == "qwen3_5_moe_text"
|
||||
and envs.SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION.get()
|
||||
and uses_cutedsl_ar_fusion()
|
||||
)
|
||||
|
||||
|
||||
def _layer_communicator_class(config: Qwen3_5TextConfig, is_nextn: bool):
|
||||
if _use_mnnvl_cutedsl_fusion(config, is_nextn):
|
||||
from sglang.srt.layers.moe.qwen35_flashinfer_fusion import (
|
||||
Qwen35FlashInferLayerCommunicator,
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import (
|
||||
CuteDSLFusionLayerCommunicator,
|
||||
)
|
||||
|
||||
return Qwen35FlashInferLayerCommunicator
|
||||
return CuteDSLFusionLayerCommunicator
|
||||
return LayerCommunicator
|
||||
|
||||
|
||||
@@ -305,22 +303,15 @@ def _select_fused_ar_input_for_linear(hidden_states, linear: nn.Module):
|
||||
|
||||
|
||||
def _finish_mlp_output(hidden_states, *, expect_deferred: bool):
|
||||
if not expect_deferred:
|
||||
if not isinstance(hidden_states, torch.Tensor):
|
||||
from sglang.srt.layers.moe.qwen35_flashinfer_fusion import (
|
||||
Qwen35MoeFinalizeHandoff,
|
||||
)
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import MoeFinalizeHandoff
|
||||
|
||||
if isinstance(hidden_states, Qwen35MoeFinalizeHandoff):
|
||||
raise RuntimeError("unexpected deferred-finalize handoff")
|
||||
if not expect_deferred:
|
||||
if isinstance(hidden_states, MoeFinalizeHandoff):
|
||||
raise RuntimeError("unexpected deferred-finalize handoff")
|
||||
hidden_states._sglang_needs_allreduce_fusion = True
|
||||
return hidden_states
|
||||
|
||||
from sglang.srt.layers.moe.qwen35_flashinfer_fusion import (
|
||||
Qwen35MoeFinalizeHandoff,
|
||||
)
|
||||
|
||||
if not isinstance(hidden_states, Qwen35MoeFinalizeHandoff):
|
||||
if not isinstance(hidden_states, MoeFinalizeHandoff):
|
||||
raise RuntimeError("Qwen3.5 expected a FlashInfer deferred-finalize handoff")
|
||||
return hidden_states
|
||||
|
||||
@@ -1199,8 +1190,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
fuse_mlp_allreduce
|
||||
and isinstance(hidden_states, torch.Tensor)
|
||||
and isinstance(self.mlp, Qwen2MoeSparseMoeBlock)
|
||||
and hasattr(self.layer_communicator, "should_use_finalize")
|
||||
and self.layer_communicator.should_use_finalize(
|
||||
and self.layer_communicator.should_defer_moe_finalize(
|
||||
forward_batch, int(hidden_states.shape[0])
|
||||
)
|
||||
)
|
||||
@@ -1637,8 +1627,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
fuse_mlp_allreduce
|
||||
and isinstance(hidden_states, torch.Tensor)
|
||||
and isinstance(self.mlp, Qwen2MoeSparseMoeBlock)
|
||||
and hasattr(self.layer_communicator, "should_use_finalize")
|
||||
and self.layer_communicator.should_use_finalize(
|
||||
and self.layer_communicator.should_defer_moe_finalize(
|
||||
forward_batch, int(hidden_states.shape[0])
|
||||
)
|
||||
)
|
||||
@@ -1825,26 +1814,20 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
"layers: "
|
||||
f"{unsupported_layers}"
|
||||
)
|
||||
from sglang.srt.layers.moe.qwen35_flashinfer_fusion import (
|
||||
Qwen35FlashInferFusionService,
|
||||
Qwen35FlashInferLayerCommunicator,
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import (
|
||||
install_cutedsl_fusion,
|
||||
)
|
||||
|
||||
self.flashinfer_mnnvl_cutedsl_fusion = Qwen35FlashInferFusionService(
|
||||
self.flashinfer_mnnvl_cutedsl_fusion = install_cutedsl_fusion(
|
||||
self.layers,
|
||||
hidden_size=config.hidden_size,
|
||||
top_k=config.num_experts_per_tok,
|
||||
rms_epsilon=config.rms_norm_eps,
|
||||
)
|
||||
for layer in self.layers:
|
||||
communicator = layer.layer_communicator
|
||||
if not isinstance(communicator, Qwen35FlashInferLayerCommunicator):
|
||||
raise RuntimeError(
|
||||
"Qwen3.5 fusion-enabled layer has the wrong communicator"
|
||||
)
|
||||
communicator.fusion_service = self.flashinfer_mnnvl_cutedsl_fusion
|
||||
logger.info(
|
||||
"Installed one Qwen3.5 FlashInfer fusion handle for %d layers",
|
||||
len(self.layers),
|
||||
# Every layer was checked above.
|
||||
can_defer_finalize=lambda layer: True,
|
||||
# The final GemmaRMSNorm closes out the last layer's handoff.
|
||||
final_norm_consumes_handoff=True,
|
||||
label="Qwen3.5",
|
||||
)
|
||||
|
||||
# Final normalization
|
||||
@@ -1879,13 +1862,6 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
logger.info(
|
||||
"Packed BF16/FP8 GDN input projection enabled for %d layers", packed
|
||||
)
|
||||
if self.flashinfer_mnnvl_cutedsl_fusion is None:
|
||||
return
|
||||
from sglang.srt.layers.moe.qwen35_flashinfer_fusion import (
|
||||
prepare_qwen35_flashinfer_fusion,
|
||||
)
|
||||
|
||||
prepare_qwen35_flashinfer_fusion(self, model_runner)
|
||||
|
||||
def set_dflash_layers_to_capture(self, layers_to_capture: list[int]):
|
||||
self.layers_to_capture = layers_to_capture
|
||||
@@ -1972,61 +1948,23 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
)
|
||||
|
||||
# The final layer has no successor to consume its deferred MoE tail.
|
||||
trace_final_norm = envs.SGLANG_TRACE_QWEN35_FINAL_NORM.get()
|
||||
use_native_final_norm = envs.SGLANG_QWEN35_NATIVE_FINAL_NORM.get()
|
||||
is_deferred_finalize = False
|
||||
if self.flashinfer_mnnvl_cutedsl_fusion is not None:
|
||||
from sglang.srt.layers.moe.qwen35_flashinfer_fusion import (
|
||||
Qwen35MoeFinalizeHandoff,
|
||||
)
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import MoeFinalizeHandoff
|
||||
|
||||
is_deferred_finalize = isinstance(hidden_states, Qwen35MoeFinalizeHandoff)
|
||||
is_deferred_finalize = isinstance(hidden_states, MoeFinalizeHandoff)
|
||||
|
||||
if is_deferred_finalize:
|
||||
if residual is None or self.flashinfer_mnnvl_cutedsl_fusion is None:
|
||||
if residual is None:
|
||||
raise RuntimeError("invalid final deferred MoE handoff")
|
||||
hidden_states, _ = self.flashinfer_mnnvl_cutedsl_fusion.finalize(
|
||||
hidden_states, residual, self.norm.gemma_weight
|
||||
handoff=hidden_states, residual=residual, gamma=self.norm.gemma_weight
|
||||
)
|
||||
elif hidden_states.shape[0] != 0:
|
||||
if trace_final_norm:
|
||||
print(
|
||||
"SGLANG_TRACE_QWEN35_FINAL_NORM "
|
||||
f"stage=pre_sync_enter hidden={tuple(hidden_states.shape)} "
|
||||
f"hidden_stride={hidden_states.stride()} "
|
||||
f"hidden_dtype={hidden_states.dtype} "
|
||||
f"hidden_contiguous={hidden_states.is_contiguous()} "
|
||||
f"residual={None if residual is None else tuple(residual.shape)} "
|
||||
f"native={use_native_final_norm}",
|
||||
flush=True,
|
||||
)
|
||||
torch.cuda.synchronize()
|
||||
print(
|
||||
"SGLANG_TRACE_QWEN35_FINAL_NORM stage=pre_sync_returned",
|
||||
flush=True,
|
||||
)
|
||||
if residual is None:
|
||||
hidden_states = (
|
||||
self.norm.forward_native(hidden_states)
|
||||
if use_native_final_norm
|
||||
else self.norm(hidden_states)
|
||||
)
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = (
|
||||
self.norm.forward_native(hidden_states, residual)
|
||||
if use_native_final_norm
|
||||
else self.norm(hidden_states, residual)
|
||||
)
|
||||
if trace_final_norm:
|
||||
print(
|
||||
"SGLANG_TRACE_QWEN35_FINAL_NORM stage=post_sync_enter",
|
||||
flush=True,
|
||||
)
|
||||
torch.cuda.synchronize()
|
||||
print(
|
||||
"SGLANG_TRACE_QWEN35_FINAL_NORM stage=post_sync_returned",
|
||||
flush=True,
|
||||
)
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
|
||||
@@ -660,9 +660,11 @@ class ForwardFlags:
|
||||
# fuse_mlp_allreduce: next residual+LN absorbs the post-MLP all-reduce.
|
||||
# mlp_reduce_scatter: postprocess will reduce-scatter (skip MLP AR).
|
||||
# flashinfer_trtllm_bypass: deepseek dual-stream graph topk bypass.
|
||||
# defer_moe_finalize: next layer's fused collective absorbs the finalize.
|
||||
"fuse_mlp_allreduce": False,
|
||||
"mlp_reduce_scatter": False,
|
||||
"flashinfer_trtllm_bypass": False,
|
||||
"defer_moe_finalize": False,
|
||||
# LayerNorm sequence parallelism region; see layers/layernorm_sp.py.
|
||||
"sp_active": False,
|
||||
}
|
||||
@@ -679,6 +681,7 @@ class ForwardFlags:
|
||||
"fuse_mlp_allreduce",
|
||||
"mlp_reduce_scatter",
|
||||
"flashinfer_trtllm_bypass",
|
||||
"defer_moe_finalize",
|
||||
"sp_active",
|
||||
}
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user