Add blackwell dual gemm fusion support for MLP (#41887)

Signed-off-by: Devashish Lal <devcode@meta.com>
This commit is contained in:
DevashishLal-CB
2026-10-09 17:26:33 +08:00
committed by GitHub
parent 5eca4d8caf
commit 4debae7534
9 changed files with 2977 additions and 13 deletions
+68 -1
View File
@@ -17,10 +17,13 @@ from sglang.kernels.spec import (
if TYPE_CHECKING:
import torch
from sglang.kernels.ops.gemm.cutedsl_dual_gemm import DualGemmQuantMode
_CUDA = frozenset({CapabilityRequirement.CUDA})
_SM90 = frozenset({CapabilityRequirement.cuda(min_sm=(9, 0), max_sm=(9, 0))})
_SM120 = frozenset({CapabilityRequirement.cuda(min_sm=(12, 0), max_sm=(12, 0))})
_SM12X = frozenset({CapabilityRequirement.cuda(min_sm=(12, 0), max_sm=(12, 9))})
_SM10X = frozenset({CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 9))})
_KDA_PACKAGE = "sglang.kernels.kda_kernels"
@@ -164,6 +167,35 @@ class Fp8ScaledMMOp(BaseFusedOp):
_FP8_SCALED_MM = register_fused_op(Fp8ScaledMMOp(), __name__, "_FP8_SCALED_MM")
register_kernel(
KernelSpec(
op="gemm.dual_gemm_swiglu",
backend=KernelBackend.CUTE_DSL,
target="sglang.kernels.ops.gemm.cutedsl_dual_gemm:dual_gemm_swiglu",
capabilities=_SM10X,
format_signature=FormatSignature(
supported_dtypes=("bfloat16", "float16"),
description="BF16/FP16 gate/up dual GEMM followed by SwiGLU",
),
description="Blackwell TMA/tcgen05 fused dual GEMM and activation.",
)
)
register_kernel(
KernelSpec(
op="gemm.dual_gemm_swiglu_fp8",
backend=KernelBackend.CUTE_DSL,
target="sglang.kernels.ops.gemm.cutedsl_dual_gemm:dual_gemm_swiglu_fp8",
capabilities=_SM10X,
format_signature=FormatSignature(
supported_dtypes=("float8_e4m3fn",),
description=(
"FP8 gate/up dual GEMM, SwiGLU, and static or dynamic "
"per-tensor or per-token FP8 activation quantization"
),
),
description="Blackwell TMA/tcgen05 fused dual GEMM and quantization.",
)
)
register_kernel(
KernelSpec(
op="gemm.bmm_fp8",
@@ -286,6 +318,39 @@ def fp8_scaled_mm(
return _FP8_SCALED_MM(mat_a, mat_b, scales_a, scales_b, out_dtype, bias)
def dual_gemm_swiglu_fp8(
x: torch.Tensor,
gate_up_weight: torch.Tensor,
x_scale: torch.Tensor,
gate_up_weight_scale: torch.Tensor,
output_scale: Optional[torch.Tensor] = None,
quant_mode: Optional["DualGemmQuantMode"] = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Fuse FP8 gate/up projections, SwiGLU, and activation quantization."""
if quant_mode is None:
from .cutedsl_dual_gemm import DualGemmQuantMode
quant_mode = DualGemmQuantMode.DYNAMIC_PER_TOKEN
return get_kernel("gemm.dual_gemm_swiglu_fp8", KernelBackend.CUTE_DSL)(
x,
gate_up_weight,
x_scale,
gate_up_weight_scale,
output_scale,
quant_mode,
)
def dual_gemm_swiglu(
x: torch.Tensor,
gate_up_weight: torch.Tensor,
) -> torch.Tensor:
"""Fuse BF16/FP16 gate/up projections followed by SwiGLU."""
return get_kernel("gemm.dual_gemm_swiglu", KernelBackend.CUTE_DSL)(
x, gate_up_weight
)
def bmm_fp8(
A: torch.Tensor,
B: torch.Tensor,
@@ -384,11 +449,13 @@ def kimi_k3_tiny_gemm(x: torch.Tensor, w: torch.Tensor) -> torch.Tensor:
__all__ = [
"kimi_k3_tiny_gemm",
"Fp8ScaledMMOp",
"bmm_fp8",
"dsv3_fused_a_gemm",
"dual_gemm_swiglu",
"dual_gemm_swiglu_fp8",
"fp8_scaled_mm",
"kimi_k3_tiny_gemm",
"n128k512_gemm_bf16",
"n32k5120_gemm_bf16",
"tiny_gemm_bf16",
File diff suppressed because it is too large Load Diff
+142
View File
@@ -0,0 +1,142 @@
"""Reusable model-layer integration for the small-batch dual GEMM kernel."""
from typing import TYPE_CHECKING, Optional
import torch
from sglang.srt.runtime_context import get_forward, get_parallel
from sglang.srt.utils import is_cuda
if TYPE_CHECKING:
from sglang.kernels.ops.gemm.cutedsl_dual_gemm import DualGemmQuantMode
from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
RowParallelLinear,
)
class DualGemm:
"""Dispatch fused gate/up projection and activation for compatible layers."""
def __init__(
self,
gate_up_proj: "MergedColumnParallelLinear",
down_proj: "RowParallelLinear",
hidden_size: int,
) -> None:
self.down_proj = down_proj
self.tp_size = get_parallel().tp_size
self.max_tokens = 0
self.mode = self._select_mode(gate_up_proj, hidden_size)
def _select_mode(
self,
gate_up_proj: "MergedColumnParallelLinear",
hidden_size: int,
) -> Optional["DualGemmQuantMode"]:
if not is_cuda():
return None
from sglang.kernels.ops.gemm.cutedsl_dual_gemm import (
MAX_DUAL_GEMM_DECODE_TOKENS,
DualGemmQuantMode,
can_use_dual_gemm,
)
from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod
gate_up_method = gate_up_proj.quant_method
if isinstance(gate_up_method, UnquantizedLinearMethod):
mode = (
DualGemmQuantMode.UNQUANT
if gate_up_proj.params_dtype in (torch.bfloat16, torch.float16)
else None
)
else:
from sglang.srt.layers.quantization.compressed_tensors.compressed_tensors import (
CompressedTensorsLinearMethod,
)
from sglang.srt.layers.quantization.compressed_tensors.schemes import (
CompressedTensorsW8A8Fp8,
)
from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod
layers = (gate_up_proj, self.down_proj)
native_fp8 = all(
isinstance(layer.quant_method, Fp8LinearMethod)
and not (
layer.quant_method.block_quant
or layer.quant_method.use_marlin
or layer.quant_method.use_mxfp8
)
for layer in layers
)
compressed_fp8 = all(
isinstance(layer.quant_method, CompressedTensorsLinearMethod)
and isinstance(layer.scheme, CompressedTensorsW8A8Fp8)
and layer.scheme.weight_block_size is None
for layer in layers
)
if gate_up_proj.params_dtype in (
torch.bfloat16,
torch.float16,
) and (native_fp8 or compressed_fp8):
mode = (
DualGemmQuantMode.DYNAMIC_PER_TOKEN
if getattr(self.down_proj, "input_scale", None) is None
else DualGemmQuantMode.STATIC_PER_TENSOR
)
else:
mode = None
local_intermediate_size = gate_up_proj.output_partition_sizes[0]
if mode is not None and can_use_dual_gemm(
MAX_DUAL_GEMM_DECODE_TOKENS,
hidden_size,
local_intermediate_size,
mode,
):
self.max_tokens = MAX_DUAL_GEMM_DECODE_TOKENS
return mode
return None
def can_run(self, x, gate_up_proj: "MergedColumnParallelLinear") -> bool:
input_tensor = x[0] if isinstance(x, tuple) else x
return (
self.mode is not None
and 1 <= input_tensor.shape[0] <= self.max_tokens
and not hasattr(gate_up_proj, "base_layer")
and not (self.tp_size > 1 and get_forward().sp_active)
)
def __call__(self, x, gate_up_proj: "MergedColumnParallelLinear"):
if not self.mode.is_quantized:
from sglang.kernels.ops.gemm import dual_gemm_swiglu
return dual_gemm_swiglu(x, gate_up_proj.weight)
from sglang.kernels.ops.gemm import dual_gemm_swiglu_fp8
if isinstance(x, tuple):
quantized_x, x_scale = x[:2]
output_dtype = x[2] if len(x) > 2 else torch.bfloat16
else:
from sglang.kernels.ops.quantization.fp8_kernel import scaled_fp8_quant
output_dtype = x.dtype
quantized_x, x_scale = scaled_fp8_quant(
x,
gate_up_proj.input_scale,
use_per_token_if_dynamic=True,
)
quantized_activation, activation_scale = dual_gemm_swiglu_fp8(
quantized_x,
gate_up_proj.weight.T,
x_scale,
gate_up_proj.weight_scale,
self.down_proj.input_scale,
quant_mode=self.mode,
)
# The down projection consumes this tuple without quantizing again. The
# original dtype controls its output type.
return quantized_activation, activation_scale, output_dtype
@@ -2220,9 +2220,9 @@ def apply_fp8_linear(
output_shape = [*input.shape[:-1], weight.shape[1]]
# A pre-quantized fp8 activation (e.g. from a fused RMSNorm+quant kernel)
# carries no original dtype: skip re-quant, reuse the supplied per-tensor
# input_scale, and emit ``pre_quant_output_dtype`` (the model's activation
# dtype, propagated by the producer) or bf16 if it was not provided.
# carries no original dtype: skip re-quant, reuse its supplied per-tensor
# or per-token input scale, and emit ``pre_quant_output_dtype`` (the model's
# activation dtype, propagated by the producer) or bf16 if omitted.
input_prequantized = input_2d.dtype in (
torch.float8_e4m3fn,
torch.float8_e4m3fnuz,
@@ -2250,11 +2250,16 @@ def apply_fp8_linear(
)
if input_prequantized:
assert input_scale is not None and (
input_scale.numel() == 1 or (_use_aiter and use_per_token_if_dynamic)
assert input_scale is not None and input_scale.numel() in (
1,
input_2d.shape[0],
)
qinput = input_2d
if channelwise_cutlass and not native_scalar_a_scale:
if (
input_scale.numel() == 1
and channelwise_cutlass
and not native_scalar_a_scale
):
# Unsupported CUTLASS epilogues require one A scale per row.
x_scale = input_scale.repeat(input_2d.shape[0]).view(-1, 1)
else:
+8 -4
View File
@@ -30,6 +30,7 @@ from sglang.srt.distributed import (
)
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.dual_gemm import DualGemm
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
@@ -65,14 +66,12 @@ from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import (
add_prefix,
is_cuda,
is_npu,
is_xpu,
make_pp_layers,
)
from sglang.utils import get_exception_traceback
_is_cuda = is_cuda()
_is_xpu = is_xpu()
logger = logging.getLogger(__name__)
@@ -118,14 +117,19 @@ class LlamaMLP(nn.Module):
f"Unsupported activation: {hidden_act}. Only silu is supported for now."
)
self.act_fn = SiluAndMul()
self.dual_gemm = DualGemm(self.gate_up_proj, self.down_proj, hidden_size)
def forward(
self,
x,
forward_batch=None,
):
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
if self.dual_gemm.can_run(x, self.gate_up_proj):
x = self.dual_gemm(x, self.gate_up_proj)
else:
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
x, _ = self.down_proj(x)
return x
+7 -2
View File
@@ -28,6 +28,7 @@ from sglang.srt.distributed import (
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.dual_gemm import DualGemm
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
@@ -100,6 +101,7 @@ class Qwen2MLP(nn.Module):
f"Unsupported activation: {hidden_act}. Only silu is supported for now."
)
self.act_fn = SiluAndMul()
self.dual_gemm = DualGemm(self.gate_up_proj, self.down_proj, hidden_size)
def forward(
self,
@@ -109,8 +111,11 @@ class Qwen2MLP(nn.Module):
if get_exec().deterministic.rl_on_policy_target is not None:
x = x.bfloat16()
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
if self.dual_gemm.can_run(x, self.gate_up_proj):
x = self.dual_gemm(x, self.gate_up_proj)
else:
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
x, _ = self.down_proj(x, forward_batch=forward_batch)
return x