Add blackwell dual gemm fusion support for MLP (#41887)
Signed-off-by: Devashish Lal <devcode@meta.com>
This commit is contained in:
@@ -17,10 +17,13 @@ from sglang.kernels.spec import (
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.gemm.cutedsl_dual_gemm import DualGemmQuantMode
|
||||
|
||||
_CUDA = frozenset({CapabilityRequirement.CUDA})
|
||||
_SM90 = frozenset({CapabilityRequirement.cuda(min_sm=(9, 0), max_sm=(9, 0))})
|
||||
_SM120 = frozenset({CapabilityRequirement.cuda(min_sm=(12, 0), max_sm=(12, 0))})
|
||||
_SM12X = frozenset({CapabilityRequirement.cuda(min_sm=(12, 0), max_sm=(12, 9))})
|
||||
_SM10X = frozenset({CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 9))})
|
||||
_KDA_PACKAGE = "sglang.kernels.kda_kernels"
|
||||
|
||||
|
||||
@@ -164,6 +167,35 @@ class Fp8ScaledMMOp(BaseFusedOp):
|
||||
|
||||
_FP8_SCALED_MM = register_fused_op(Fp8ScaledMMOp(), __name__, "_FP8_SCALED_MM")
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="gemm.dual_gemm_swiglu",
|
||||
backend=KernelBackend.CUTE_DSL,
|
||||
target="sglang.kernels.ops.gemm.cutedsl_dual_gemm:dual_gemm_swiglu",
|
||||
capabilities=_SM10X,
|
||||
format_signature=FormatSignature(
|
||||
supported_dtypes=("bfloat16", "float16"),
|
||||
description="BF16/FP16 gate/up dual GEMM followed by SwiGLU",
|
||||
),
|
||||
description="Blackwell TMA/tcgen05 fused dual GEMM and activation.",
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="gemm.dual_gemm_swiglu_fp8",
|
||||
backend=KernelBackend.CUTE_DSL,
|
||||
target="sglang.kernels.ops.gemm.cutedsl_dual_gemm:dual_gemm_swiglu_fp8",
|
||||
capabilities=_SM10X,
|
||||
format_signature=FormatSignature(
|
||||
supported_dtypes=("float8_e4m3fn",),
|
||||
description=(
|
||||
"FP8 gate/up dual GEMM, SwiGLU, and static or dynamic "
|
||||
"per-tensor or per-token FP8 activation quantization"
|
||||
),
|
||||
),
|
||||
description="Blackwell TMA/tcgen05 fused dual GEMM and quantization.",
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="gemm.bmm_fp8",
|
||||
@@ -286,6 +318,39 @@ def fp8_scaled_mm(
|
||||
return _FP8_SCALED_MM(mat_a, mat_b, scales_a, scales_b, out_dtype, bias)
|
||||
|
||||
|
||||
def dual_gemm_swiglu_fp8(
|
||||
x: torch.Tensor,
|
||||
gate_up_weight: torch.Tensor,
|
||||
x_scale: torch.Tensor,
|
||||
gate_up_weight_scale: torch.Tensor,
|
||||
output_scale: Optional[torch.Tensor] = None,
|
||||
quant_mode: Optional["DualGemmQuantMode"] = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Fuse FP8 gate/up projections, SwiGLU, and activation quantization."""
|
||||
if quant_mode is None:
|
||||
from .cutedsl_dual_gemm import DualGemmQuantMode
|
||||
|
||||
quant_mode = DualGemmQuantMode.DYNAMIC_PER_TOKEN
|
||||
return get_kernel("gemm.dual_gemm_swiglu_fp8", KernelBackend.CUTE_DSL)(
|
||||
x,
|
||||
gate_up_weight,
|
||||
x_scale,
|
||||
gate_up_weight_scale,
|
||||
output_scale,
|
||||
quant_mode,
|
||||
)
|
||||
|
||||
|
||||
def dual_gemm_swiglu(
|
||||
x: torch.Tensor,
|
||||
gate_up_weight: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Fuse BF16/FP16 gate/up projections followed by SwiGLU."""
|
||||
return get_kernel("gemm.dual_gemm_swiglu", KernelBackend.CUTE_DSL)(
|
||||
x, gate_up_weight
|
||||
)
|
||||
|
||||
|
||||
def bmm_fp8(
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
@@ -384,11 +449,13 @@ def kimi_k3_tiny_gemm(x: torch.Tensor, w: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
|
||||
__all__ = [
|
||||
"kimi_k3_tiny_gemm",
|
||||
"Fp8ScaledMMOp",
|
||||
"bmm_fp8",
|
||||
"dsv3_fused_a_gemm",
|
||||
"dual_gemm_swiglu",
|
||||
"dual_gemm_swiglu_fp8",
|
||||
"fp8_scaled_mm",
|
||||
"kimi_k3_tiny_gemm",
|
||||
"n128k512_gemm_bf16",
|
||||
"n32k5120_gemm_bf16",
|
||||
"tiny_gemm_bf16",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,142 @@
|
||||
"""Reusable model-layer integration for the small-batch dual GEMM kernel."""
|
||||
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.runtime_context import get_forward, get_parallel
|
||||
from sglang.srt.utils import is_cuda
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.kernels.ops.gemm.cutedsl_dual_gemm import DualGemmQuantMode
|
||||
from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
|
||||
|
||||
class DualGemm:
|
||||
"""Dispatch fused gate/up projection and activation for compatible layers."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
gate_up_proj: "MergedColumnParallelLinear",
|
||||
down_proj: "RowParallelLinear",
|
||||
hidden_size: int,
|
||||
) -> None:
|
||||
self.down_proj = down_proj
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.max_tokens = 0
|
||||
self.mode = self._select_mode(gate_up_proj, hidden_size)
|
||||
|
||||
def _select_mode(
|
||||
self,
|
||||
gate_up_proj: "MergedColumnParallelLinear",
|
||||
hidden_size: int,
|
||||
) -> Optional["DualGemmQuantMode"]:
|
||||
if not is_cuda():
|
||||
return None
|
||||
|
||||
from sglang.kernels.ops.gemm.cutedsl_dual_gemm import (
|
||||
MAX_DUAL_GEMM_DECODE_TOKENS,
|
||||
DualGemmQuantMode,
|
||||
can_use_dual_gemm,
|
||||
)
|
||||
from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod
|
||||
|
||||
gate_up_method = gate_up_proj.quant_method
|
||||
if isinstance(gate_up_method, UnquantizedLinearMethod):
|
||||
mode = (
|
||||
DualGemmQuantMode.UNQUANT
|
||||
if gate_up_proj.params_dtype in (torch.bfloat16, torch.float16)
|
||||
else None
|
||||
)
|
||||
else:
|
||||
from sglang.srt.layers.quantization.compressed_tensors.compressed_tensors import (
|
||||
CompressedTensorsLinearMethod,
|
||||
)
|
||||
from sglang.srt.layers.quantization.compressed_tensors.schemes import (
|
||||
CompressedTensorsW8A8Fp8,
|
||||
)
|
||||
from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod
|
||||
|
||||
layers = (gate_up_proj, self.down_proj)
|
||||
native_fp8 = all(
|
||||
isinstance(layer.quant_method, Fp8LinearMethod)
|
||||
and not (
|
||||
layer.quant_method.block_quant
|
||||
or layer.quant_method.use_marlin
|
||||
or layer.quant_method.use_mxfp8
|
||||
)
|
||||
for layer in layers
|
||||
)
|
||||
compressed_fp8 = all(
|
||||
isinstance(layer.quant_method, CompressedTensorsLinearMethod)
|
||||
and isinstance(layer.scheme, CompressedTensorsW8A8Fp8)
|
||||
and layer.scheme.weight_block_size is None
|
||||
for layer in layers
|
||||
)
|
||||
if gate_up_proj.params_dtype in (
|
||||
torch.bfloat16,
|
||||
torch.float16,
|
||||
) and (native_fp8 or compressed_fp8):
|
||||
mode = (
|
||||
DualGemmQuantMode.DYNAMIC_PER_TOKEN
|
||||
if getattr(self.down_proj, "input_scale", None) is None
|
||||
else DualGemmQuantMode.STATIC_PER_TENSOR
|
||||
)
|
||||
else:
|
||||
mode = None
|
||||
|
||||
local_intermediate_size = gate_up_proj.output_partition_sizes[0]
|
||||
if mode is not None and can_use_dual_gemm(
|
||||
MAX_DUAL_GEMM_DECODE_TOKENS,
|
||||
hidden_size,
|
||||
local_intermediate_size,
|
||||
mode,
|
||||
):
|
||||
self.max_tokens = MAX_DUAL_GEMM_DECODE_TOKENS
|
||||
return mode
|
||||
return None
|
||||
|
||||
def can_run(self, x, gate_up_proj: "MergedColumnParallelLinear") -> bool:
|
||||
input_tensor = x[0] if isinstance(x, tuple) else x
|
||||
return (
|
||||
self.mode is not None
|
||||
and 1 <= input_tensor.shape[0] <= self.max_tokens
|
||||
and not hasattr(gate_up_proj, "base_layer")
|
||||
and not (self.tp_size > 1 and get_forward().sp_active)
|
||||
)
|
||||
|
||||
def __call__(self, x, gate_up_proj: "MergedColumnParallelLinear"):
|
||||
if not self.mode.is_quantized:
|
||||
from sglang.kernels.ops.gemm import dual_gemm_swiglu
|
||||
|
||||
return dual_gemm_swiglu(x, gate_up_proj.weight)
|
||||
|
||||
from sglang.kernels.ops.gemm import dual_gemm_swiglu_fp8
|
||||
|
||||
if isinstance(x, tuple):
|
||||
quantized_x, x_scale = x[:2]
|
||||
output_dtype = x[2] if len(x) > 2 else torch.bfloat16
|
||||
else:
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import scaled_fp8_quant
|
||||
|
||||
output_dtype = x.dtype
|
||||
quantized_x, x_scale = scaled_fp8_quant(
|
||||
x,
|
||||
gate_up_proj.input_scale,
|
||||
use_per_token_if_dynamic=True,
|
||||
)
|
||||
|
||||
quantized_activation, activation_scale = dual_gemm_swiglu_fp8(
|
||||
quantized_x,
|
||||
gate_up_proj.weight.T,
|
||||
x_scale,
|
||||
gate_up_proj.weight_scale,
|
||||
self.down_proj.input_scale,
|
||||
quant_mode=self.mode,
|
||||
)
|
||||
# The down projection consumes this tuple without quantizing again. The
|
||||
# original dtype controls its output type.
|
||||
return quantized_activation, activation_scale, output_dtype
|
||||
@@ -2220,9 +2220,9 @@ def apply_fp8_linear(
|
||||
output_shape = [*input.shape[:-1], weight.shape[1]]
|
||||
|
||||
# A pre-quantized fp8 activation (e.g. from a fused RMSNorm+quant kernel)
|
||||
# carries no original dtype: skip re-quant, reuse the supplied per-tensor
|
||||
# input_scale, and emit ``pre_quant_output_dtype`` (the model's activation
|
||||
# dtype, propagated by the producer) or bf16 if it was not provided.
|
||||
# carries no original dtype: skip re-quant, reuse its supplied per-tensor
|
||||
# or per-token input scale, and emit ``pre_quant_output_dtype`` (the model's
|
||||
# activation dtype, propagated by the producer) or bf16 if omitted.
|
||||
input_prequantized = input_2d.dtype in (
|
||||
torch.float8_e4m3fn,
|
||||
torch.float8_e4m3fnuz,
|
||||
@@ -2250,11 +2250,16 @@ def apply_fp8_linear(
|
||||
)
|
||||
|
||||
if input_prequantized:
|
||||
assert input_scale is not None and (
|
||||
input_scale.numel() == 1 or (_use_aiter and use_per_token_if_dynamic)
|
||||
assert input_scale is not None and input_scale.numel() in (
|
||||
1,
|
||||
input_2d.shape[0],
|
||||
)
|
||||
qinput = input_2d
|
||||
if channelwise_cutlass and not native_scalar_a_scale:
|
||||
if (
|
||||
input_scale.numel() == 1
|
||||
and channelwise_cutlass
|
||||
and not native_scalar_a_scale
|
||||
):
|
||||
# Unsupported CUTLASS epilogues require one A scale per row.
|
||||
x_scale = input_scale.repeat(input_2d.shape[0]).view(-1, 1)
|
||||
else:
|
||||
|
||||
@@ -30,6 +30,7 @@ from sglang.srt.distributed import (
|
||||
)
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.dual_gemm import DualGemm
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
@@ -65,14 +66,12 @@ from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import (
|
||||
add_prefix,
|
||||
is_cuda,
|
||||
is_npu,
|
||||
is_xpu,
|
||||
make_pp_layers,
|
||||
)
|
||||
from sglang.utils import get_exception_traceback
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
_is_xpu = is_xpu()
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -118,14 +117,19 @@ class LlamaMLP(nn.Module):
|
||||
f"Unsupported activation: {hidden_act}. Only silu is supported for now."
|
||||
)
|
||||
self.act_fn = SiluAndMul()
|
||||
self.dual_gemm = DualGemm(self.gate_up_proj, self.down_proj, hidden_size)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x,
|
||||
forward_batch=None,
|
||||
):
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
if self.dual_gemm.can_run(x, self.gate_up_proj):
|
||||
x = self.dual_gemm(x, self.gate_up_proj)
|
||||
else:
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ from sglang.srt.distributed import (
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.dual_gemm import DualGemm
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
@@ -100,6 +101,7 @@ class Qwen2MLP(nn.Module):
|
||||
f"Unsupported activation: {hidden_act}. Only silu is supported for now."
|
||||
)
|
||||
self.act_fn = SiluAndMul()
|
||||
self.dual_gemm = DualGemm(self.gate_up_proj, self.down_proj, hidden_size)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -109,8 +111,11 @@ class Qwen2MLP(nn.Module):
|
||||
if get_exec().deterministic.rl_on_policy_target is not None:
|
||||
x = x.bfloat16()
|
||||
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
if self.dual_gemm.can_run(x, self.gate_up_proj):
|
||||
x = self.dual_gemm(x, self.gate_up_proj)
|
||||
else:
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
x, _ = self.down_proj(x, forward_batch=forward_batch)
|
||||
return x
|
||||
|
||||
|
||||
Reference in New Issue
Block a user