[Perf] mlp: Fuse dense SwiGLU with block-FP8 quantization on decode (#45219)

Co-authored-by: Jayden Yu <jaydenyu@fb.com>
Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
Jayden Yu
2026-10-23 07:49:39 +08:00
committed by GitHub
co-authored by Jayden Yu Xiaoyu Zhang
parent 5849ff5337
commit bb9e9d5632
11 changed files with 878 additions and 29 deletions
@@ -238,7 +238,8 @@ template <
bool kUe8m0_,
bool kRowMajor_,
bool kAligned_,
bool kFuseSiluAndMul_>
bool kFuseSiluAndMul_,
bool kDenseSiluAndMul = false>
struct QuantTrait {
// rename
using InputType = InputType_;
@@ -253,6 +254,7 @@ struct QuantTrait {
static constexpr uint32_t kNumLanes = kGroupSize / kVecSize;
static_assert(sizeof(InputType) == 2 || sizeof(InputType) == 4, "inputs must be 16-bit (bf16/fp16) or fp32");
static_assert(sizeof(InputType) == 2 || !kFuseSiluAndMul, "fp32 inputs do not implement the fused silu");
static_assert(!kDenseSiluAndMul || (kFuseSiluAndMul && kGroupSize == 128 && std::is_same_v<QuantType, fp8_e4m3_t>));
static_assert(16 <= kGroupSize && kGroupSize <= 256, "supported group sizes are 16..256");
static_assert(kGroupSize % kVecSize == 0 && 1 <= kNumLanes && kNumLanes <= device::kWarpThreads);
static_assert(!kUe8m0 || std::is_same_v<QuantType, fp8_e4m3_t>, "ue8m0 scales imply fp8 output");
@@ -302,8 +304,15 @@ struct QuantTrait {
#pragma unroll
for (uint32_t i = 0; i < kVecSize / 2; ++i) {
const auto gate = cast<float2>(in[i]);
const auto act = cast<T2>(float2{detail::silu(gate.x), detail::silu(gate.y)});
in[i] = __hmul2(act, up[i]);
if constexpr (kDenseSiluAndMul) {
// Match dense activation.cuh: round the product, not SiLU, to T.
const auto up_f32 = cast<float2>(up[i]);
in[i] = cast<T2>(
float2{(gate.x / (1.0f + expf(-gate.x))) * up_f32.x, (gate.y / (1.0f + expf(-gate.y))) * up_f32.y});
} else {
const auto act = cast<T2>(float2{detail::silu(gate.x), detail::silu(gate.y)});
in[i] = __hmul2(act, up[i]);
}
}
}
@@ -342,7 +351,9 @@ struct QuantTrait {
} else {
// fp32 scale: multiply in fp32 (hmul2 brings too much precision loss)
scale_inv = raw_scale;
const float quant_scale = kMaxValue / amax;
// Dense mode uses activation's compiler flags, but quantization must
// retain the fast division used by the standalone quantizer.
const float quant_scale = kDenseSiluAndMul ? __fdividef(kMaxValue, amax) : kMaxValue / amax;
const float2 quant_scale2 = {quant_scale, quant_scale};
#pragma unroll
for (uint32_t i = 0; i < kVecSize / 2; ++i) {
@@ -621,9 +632,11 @@ template <
bool kRowMajor,
bool kAligned,
bool kFuseSiluAndMul,
bool kUsePDL>
bool kUsePDL,
bool kDenseSiluAndMul = false>
struct PerTokenGroupQuantFlatKernel {
using Trait = QuantTrait<InputType, QuantType, kGroupSize, kUe8m0, kRowMajor, kAligned, kFuseSiluAndMul>;
using Trait =
QuantTrait<InputType, QuantType, kGroupSize, kUe8m0, kRowMajor, kAligned, kFuseSiluAndMul, kDenseSiluAndMul>;
static void run(tvm::ffi::TensorView input, tvm::ffi::TensorView output_q, tvm::ffi::TensorView output_s) {
const auto ctx = build_quant_context<Trait, /*kMasked=*/false>(input, output_q, output_s);
@@ -1,11 +1,12 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Optional, Tuple
from typing import TYPE_CHECKING, Literal, Optional, Tuple
import torch
from sglang.kernels.jit.utils import (
cache_once,
get_activation_cuda_cflags,
is_arch_support_pdl,
load_jit,
make_cpp_args,
@@ -33,10 +34,26 @@ def _jit_module(
masked_layout: bool,
use_pdl: bool,
sorted_layout: bool = False,
silu_and_mul_mode: str = "legacy",
) -> Module:
assert in_dtype in _SUPPORTED_INPUT_DTYPES
assert out_dtype in _SUPPORTED_OUTPUT_DTYPES
assert group_size in _SUPPORTED_GROUP_SIZES
if silu_and_mul_mode not in ("legacy", "dense"):
raise ValueError(f"Unknown silu_and_mul_mode: {silu_and_mul_mode}")
dense = silu_and_mul_mode == "dense"
if dense and not (
fuse_silu_and_mul
and in_dtype in (torch.bfloat16, torch.float16)
and out_dtype == torch.float8_e4m3fn
and group_size == 128
and not masked_layout
and not sorted_layout
):
raise ValueError(
"dense mode requires flat BF16/FP16 SiLU+mul, FP8 output and group_size=128"
)
# Only the flat dense variant adds a flag; legacy launcher arguments stay intact.
trait_args = make_cpp_args(
in_dtype,
out_dtype,
@@ -46,6 +63,7 @@ def _jit_module(
aligned,
fuse_silu_and_mul,
use_pdl,
*([True] if dense else []),
)
assert not (masked_layout and sorted_layout)
layout = "masked" if masked_layout else ("sorted" if sorted_layout else "flat")
@@ -56,7 +74,9 @@ def _jit_module(
layout,
cuda_files=["gemm/per_token_group_quant.cuh"],
cuda_wrappers=[("per_token_group_quant", f"{launcher}<{trait_args}>::run")],
extra_cuda_cflags=["--use_fast_math"],
extra_cuda_cflags=get_activation_cuda_cflags()
if dense
else ["--use_fast_math"],
)
@@ -97,6 +117,7 @@ def _per_token_group_quant_custom_op(
sorted_token_ids: Optional[torch.Tensor] = None,
num_tokens_post_padded: Optional[torch.Tensor] = None,
num_valid_tokens: int = 0,
silu_and_mul_mode: str = "legacy",
) -> None:
num_groups = output_q.shape[-1] // group_size
row_major, aligned = _infer_scale_layout(output_s, scale_ue8m0, num_groups)
@@ -111,6 +132,7 @@ def _per_token_group_quant_custom_op(
masked_m is not None,
is_arch_support_pdl(),
sorted_token_ids is not None,
silu_and_mul_mode,
)
if masked_m is not None:
module.per_token_group_quant(
@@ -186,6 +208,7 @@ def per_token_group_quant(
sorted_token_ids: Optional[torch.Tensor] = None,
num_tokens_post_padded: Optional[torch.Tensor] = None,
num_valid_tokens: int = 0,
silu_and_mul_mode: Literal["legacy", "dense"] = "legacy",
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Per-token-group quantization. Returns ``(output_q, output_s)``.
@@ -207,6 +230,11 @@ def per_token_group_quant(
int32 contiguous -> row-major UE8M0 bytes packed 4-per-int32
The packed layouts require ``scale_ue8m0=True``.
With ``fuse_silu_and_mul=True``, ``silu_and_mul_mode="dense"`` matches the
dense CUDA activation: FP32 SiLU and multiply, then one source-dtype round
before quantizing. The default retains the existing early SiLU rounding.
Dense mode supports flat BF16/FP16 inputs, FP8 output and groups of 128.
``expected_m`` (masked only) is an optional expected-tokens-per-expert hint.
``sorted_token_ids`` selects the sorted layout: ``input`` rows follow
@@ -242,5 +270,6 @@ def per_token_group_quant(
sorted_token_ids=sorted_token_ids,
num_tokens_post_padded=num_tokens_post_padded,
num_valid_tokens=num_valid_tokens,
silu_and_mul_mode=silu_and_mul_mode,
)
return output_q, output_s
@@ -0,0 +1,119 @@
"""Dense activation/quantization fusion for the block-FP8 DeepGEMM path."""
from __future__ import annotations
from functools import partial
from typing import TYPE_CHECKING
import torch
from sglang.kernels.fused_op import get_fused_op_backend
from sglang.srt.runtime_context import (
get_exec,
get_forward,
get_lora,
get_parallel,
get_platform,
)
from sglang.srt.utils import is_cuda
if TYPE_CHECKING:
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.linear import RowParallelLinear
class DenseSiluAndMulQuant:
"""Keep the normal activation unless down_proj can consume our FP8 tuple."""
def __init__(self, down_proj: RowParallelLinear) -> None:
self.enabled = False
self.down_proj = down_proj
self.quant_method = down_proj.quant_method
if not (is_cuda() and get_platform().is_sm100 and get_parallel().tp_size == 1):
return
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.linear import RowParallelLinear
from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod
from sglang.srt.layers.quantization.fp8_utils import (
_deepgemm_block_fp8_supported,
deepgemm_w8a8_block_fp8_linear_with_fallback,
)
method = self.quant_method
if not (
type(down_proj) is RowParallelLinear
and type(method) is Fp8LinearMethod
and method.weight_block_size == [128, 128]
and method.quant_config.activation_scheme == "dynamic"
and not (method.use_marlin or method.use_mxfp8 or method.block_fp8_as_mxfp8)
and deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
and down_proj.input_is_parallel
):
return
backend = method.w8a8_block_fp8_linear
if isinstance(backend, partial):
if backend.args or set(backend.keywords) - {"act_scale_ue8m0"}:
return
backend = backend.func
# The supported loader preserves matrix geometry, not tensor identity/device.
self.weight_shape = (down_proj.output_size, down_proj.input_size_per_partition)
self.enabled = (
backend is deepgemm_w8a8_block_fp8_linear_with_fallback
and _deepgemm_block_fp8_supported(
self.weight_shape, method.weight_block_size, torch.bfloat16
)
)
def forward_decode(
self,
x: torch.Tensor,
act_fn: SiluAndMul,
down_proj: RowParallelLinear,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""Try fusion for decode; callers must bypass this helper in other modes."""
if not self.enabled or torch.compiler.is_compiling():
return act_fn(x)
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
from sglang.srt.layers.activation import SiluAndMul
deterministic = get_exec().deterministic
if (
down_proj is not self.down_proj
or down_proj.quant_method is not self.quant_method
or down_proj.weight.shape != self.weight_shape
or type(act_fn) is not SiluAndMul
or (act_fn._forward_method or act_fn._resolve_forward_method())
!= act_fn.forward_cuda
or act_fn.is_torch_compile
or get_fused_op_backend() is not None
or deterministic.enable_deterministic_inference
or deterministic.rl_on_policy_target is not None
or is_batch_invariant_mode_enabled()
or get_lora().enable_lora
or get_forward().sp_active
or x.dtype != torch.bfloat16
or not x.is_cuda
or x.ndim != 2
or x.shape[0] == 0
or not x.is_contiguous()
or x.data_ptr() % 32 != 0
or x.stride(0) % 16 != 0
or x.device != down_proj.weight.device
or x.shape[1] != 2 * self.weight_shape[1]
):
return act_fn(x)
from sglang.kernels.ops.quantization.per_token_group_quant import (
per_token_group_quant,
)
return per_token_group_quant(
x,
group_size=128,
scale_ue8m0=True,
column_major_scales=True,
fuse_silu_and_mul=True,
silu_and_mul_mode="dense",
)
@@ -1453,6 +1453,25 @@ def _deepgemm_block_fp8_supported(
return list(block_size) == [128, 128] and n % 64 == 0 and k % 128 == 0
def _deepgemm_packed_input_valid(input, weight, input_scale, weight_scale) -> bool:
"""Validate the group-128, column-major TMA layout for packed activations."""
if input.ndim != 2 or weight.ndim != 2:
return False
m, k = input.shape
packed_groups = ceil_div(k // 128, 4)
padded_m = ceil_align(m, 4)
return (
input.is_cuda
and input.is_contiguous()
and k == weight.shape[1]
and input.device == weight.device == input_scale.device == weight_scale.device
and input_scale.shape == (m, packed_groups)
and input_scale.stride() == (1, padded_m)
and input.data_ptr() % 16 == 0
and input_scale.data_ptr() % 16 == 0
)
def deepgemm_w8a8_block_fp8_linear_with_fallback(
input: torch.Tensor,
weight: torch.Tensor,
@@ -1463,18 +1482,36 @@ def deepgemm_w8a8_block_fp8_linear_with_fallback(
act_scale_ue8m0: bool = False,
weight_bf16: Optional[torch.Tensor] = None,
) -> torch.Tensor:
if input_scale is not None:
# Block32 pre-quantized input reaches Triton below on UE8M0 DeepGEMM.
assert (
list(block_size) == [32, 32] or not deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
)
assert input.dtype == torch.float8_e4m3fn
output_dtype = torch.bfloat16 if input_scale is not None else input.dtype
# TODO: https://github.com/sgl-project/sglang/pull/6890#issuecomment-2943395737
if not _deepgemm_block_fp8_supported(
deepgemm_supported = weight.ndim == 2 and _deepgemm_block_fp8_supported(
weight.shape, block_size, output_dtype
) or _ue8m0_deepgemm_skips_block32(block_size, weight_scale, input_scale):
)
if input_scale is not None:
if input_scale.dtype == torch.int32:
# Packed activation scales cannot go through the Triton fallback.
if not (
deepgemm_supported
and deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
and list(block_size) == [128, 128]
and _deepgemm_packed_input_valid(
input, weight, input_scale, weight_scale
)
):
raise ValueError(
"Pre-quantized DeepGEMM block-FP8 input requires supported "
"[128, 128] weights and TMA-aligned, column-major packed UE8M0 scales"
)
else:
# Block32 pre-quantized input uses the Triton fallback.
assert (
list(block_size) == [32, 32]
or not deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
)
assert input.dtype == torch.float8_e4m3fn
# TODO: https://github.com/sgl-project/sglang/pull/6890#issuecomment-2943395737
if not deepgemm_supported or _ue8m0_deepgemm_skips_block32(
block_size, weight_scale, input_scale
):
# fall back to triton
# If weight_scale is in UE8M0 packed format (int32), convert back to float32
# UE8M0 format has shape (N, K//block_k//4) with dtype int32
@@ -1495,8 +1532,8 @@ def deepgemm_w8a8_block_fp8_linear_with_fallback(
)
if input_scale is not None:
# Pre-quantized activations retain their per-token group size and
# fp32 scales. DeepGEMM aligns scale storage for TMA when needed.
# Pre-quantized activations retain their group size and scale format.
# DeepGEMM aligns fp32 scale storage for TMA when needed.
input_2d = input.view(-1, input.shape[-1])
output = w8a8_block_fp8_matmul_deepgemm(
input_2d,
+9 -2
View File
@@ -49,6 +49,7 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
from sglang.srt.layers.pooler import Pooler, PoolingType
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.fp8_activation import DenseSiluAndMulQuant
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
@@ -118,6 +119,7 @@ class LlamaMLP(nn.Module):
)
self.act_fn = SiluAndMul()
self.dual_gemm = DualGemm(self.gate_up_proj, self.down_proj, hidden_size)
self.activation_quant = DenseSiluAndMulQuant(self.down_proj)
def forward(
self,
@@ -128,7 +130,12 @@ class LlamaMLP(nn.Module):
x = self.dual_gemm(x, self.gate_up_proj)
else:
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
if forward_batch is not None and forward_batch.forward_mode.is_decode():
x = self.activation_quant.forward_decode(
gate_up, self.act_fn, self.down_proj
)
else:
x = self.act_fn(gate_up)
x, _ = self.down_proj(x)
return x
@@ -404,7 +411,7 @@ class LlamaDecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
hidden_states = self.mlp(hidden_states)
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
return self.ffn_boundary.finish(hidden_states, forward_batch)
+9 -2
View File
@@ -46,6 +46,7 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.pooler import Pooler, PoolingType
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.fp8_activation import DenseSiluAndMulQuant
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
@@ -102,6 +103,7 @@ class Qwen2MLP(nn.Module):
)
self.act_fn = SiluAndMul()
self.dual_gemm = DualGemm(self.gate_up_proj, self.down_proj, hidden_size)
self.activation_quant = DenseSiluAndMulQuant(self.down_proj)
def forward(
self,
@@ -115,7 +117,12 @@ class Qwen2MLP(nn.Module):
x = self.dual_gemm(x, self.gate_up_proj)
else:
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
if forward_batch is not None and forward_batch.forward_mode.is_decode():
x = self.activation_quant.forward_decode(
gate_up, self.act_fn, self.down_proj
)
else:
x = self.act_fn(gate_up)
x, _ = self.down_proj(x, forward_batch=forward_batch)
return x
@@ -339,7 +346,7 @@ class Qwen2DecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
hidden_states = self.mlp(hidden_states)
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
return self.ffn_boundary.finish(hidden_states, forward_batch)