[Perf] mlp: Fuse dense SwiGLU with block-FP8 quantization on decode (#45219)
Co-authored-by: Jayden Yu <jaydenyu@fb.com> Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
co-authored by
Jayden Yu
Xiaoyu Zhang
parent
5849ff5337
commit
bb9e9d5632
@@ -238,7 +238,8 @@ template <
|
||||
bool kUe8m0_,
|
||||
bool kRowMajor_,
|
||||
bool kAligned_,
|
||||
bool kFuseSiluAndMul_>
|
||||
bool kFuseSiluAndMul_,
|
||||
bool kDenseSiluAndMul = false>
|
||||
struct QuantTrait {
|
||||
// rename
|
||||
using InputType = InputType_;
|
||||
@@ -253,6 +254,7 @@ struct QuantTrait {
|
||||
static constexpr uint32_t kNumLanes = kGroupSize / kVecSize;
|
||||
static_assert(sizeof(InputType) == 2 || sizeof(InputType) == 4, "inputs must be 16-bit (bf16/fp16) or fp32");
|
||||
static_assert(sizeof(InputType) == 2 || !kFuseSiluAndMul, "fp32 inputs do not implement the fused silu");
|
||||
static_assert(!kDenseSiluAndMul || (kFuseSiluAndMul && kGroupSize == 128 && std::is_same_v<QuantType, fp8_e4m3_t>));
|
||||
static_assert(16 <= kGroupSize && kGroupSize <= 256, "supported group sizes are 16..256");
|
||||
static_assert(kGroupSize % kVecSize == 0 && 1 <= kNumLanes && kNumLanes <= device::kWarpThreads);
|
||||
static_assert(!kUe8m0 || std::is_same_v<QuantType, fp8_e4m3_t>, "ue8m0 scales imply fp8 output");
|
||||
@@ -302,8 +304,15 @@ struct QuantTrait {
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kVecSize / 2; ++i) {
|
||||
const auto gate = cast<float2>(in[i]);
|
||||
const auto act = cast<T2>(float2{detail::silu(gate.x), detail::silu(gate.y)});
|
||||
in[i] = __hmul2(act, up[i]);
|
||||
if constexpr (kDenseSiluAndMul) {
|
||||
// Match dense activation.cuh: round the product, not SiLU, to T.
|
||||
const auto up_f32 = cast<float2>(up[i]);
|
||||
in[i] = cast<T2>(
|
||||
float2{(gate.x / (1.0f + expf(-gate.x))) * up_f32.x, (gate.y / (1.0f + expf(-gate.y))) * up_f32.y});
|
||||
} else {
|
||||
const auto act = cast<T2>(float2{detail::silu(gate.x), detail::silu(gate.y)});
|
||||
in[i] = __hmul2(act, up[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -342,7 +351,9 @@ struct QuantTrait {
|
||||
} else {
|
||||
// fp32 scale: multiply in fp32 (hmul2 brings too much precision loss)
|
||||
scale_inv = raw_scale;
|
||||
const float quant_scale = kMaxValue / amax;
|
||||
// Dense mode uses activation's compiler flags, but quantization must
|
||||
// retain the fast division used by the standalone quantizer.
|
||||
const float quant_scale = kDenseSiluAndMul ? __fdividef(kMaxValue, amax) : kMaxValue / amax;
|
||||
const float2 quant_scale2 = {quant_scale, quant_scale};
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kVecSize / 2; ++i) {
|
||||
@@ -621,9 +632,11 @@ template <
|
||||
bool kRowMajor,
|
||||
bool kAligned,
|
||||
bool kFuseSiluAndMul,
|
||||
bool kUsePDL>
|
||||
bool kUsePDL,
|
||||
bool kDenseSiluAndMul = false>
|
||||
struct PerTokenGroupQuantFlatKernel {
|
||||
using Trait = QuantTrait<InputType, QuantType, kGroupSize, kUe8m0, kRowMajor, kAligned, kFuseSiluAndMul>;
|
||||
using Trait =
|
||||
QuantTrait<InputType, QuantType, kGroupSize, kUe8m0, kRowMajor, kAligned, kFuseSiluAndMul, kDenseSiluAndMul>;
|
||||
|
||||
static void run(tvm::ffi::TensorView input, tvm::ffi::TensorView output_q, tvm::ffi::TensorView output_s) {
|
||||
const auto ctx = build_quant_context<Trait, /*kMasked=*/false>(input, output_q, output_s);
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Optional, Tuple
|
||||
from typing import TYPE_CHECKING, Literal, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import (
|
||||
cache_once,
|
||||
get_activation_cuda_cflags,
|
||||
is_arch_support_pdl,
|
||||
load_jit,
|
||||
make_cpp_args,
|
||||
@@ -33,10 +34,26 @@ def _jit_module(
|
||||
masked_layout: bool,
|
||||
use_pdl: bool,
|
||||
sorted_layout: bool = False,
|
||||
silu_and_mul_mode: str = "legacy",
|
||||
) -> Module:
|
||||
assert in_dtype in _SUPPORTED_INPUT_DTYPES
|
||||
assert out_dtype in _SUPPORTED_OUTPUT_DTYPES
|
||||
assert group_size in _SUPPORTED_GROUP_SIZES
|
||||
if silu_and_mul_mode not in ("legacy", "dense"):
|
||||
raise ValueError(f"Unknown silu_and_mul_mode: {silu_and_mul_mode}")
|
||||
dense = silu_and_mul_mode == "dense"
|
||||
if dense and not (
|
||||
fuse_silu_and_mul
|
||||
and in_dtype in (torch.bfloat16, torch.float16)
|
||||
and out_dtype == torch.float8_e4m3fn
|
||||
and group_size == 128
|
||||
and not masked_layout
|
||||
and not sorted_layout
|
||||
):
|
||||
raise ValueError(
|
||||
"dense mode requires flat BF16/FP16 SiLU+mul, FP8 output and group_size=128"
|
||||
)
|
||||
# Only the flat dense variant adds a flag; legacy launcher arguments stay intact.
|
||||
trait_args = make_cpp_args(
|
||||
in_dtype,
|
||||
out_dtype,
|
||||
@@ -46,6 +63,7 @@ def _jit_module(
|
||||
aligned,
|
||||
fuse_silu_and_mul,
|
||||
use_pdl,
|
||||
*([True] if dense else []),
|
||||
)
|
||||
assert not (masked_layout and sorted_layout)
|
||||
layout = "masked" if masked_layout else ("sorted" if sorted_layout else "flat")
|
||||
@@ -56,7 +74,9 @@ def _jit_module(
|
||||
layout,
|
||||
cuda_files=["gemm/per_token_group_quant.cuh"],
|
||||
cuda_wrappers=[("per_token_group_quant", f"{launcher}<{trait_args}>::run")],
|
||||
extra_cuda_cflags=["--use_fast_math"],
|
||||
extra_cuda_cflags=get_activation_cuda_cflags()
|
||||
if dense
|
||||
else ["--use_fast_math"],
|
||||
)
|
||||
|
||||
|
||||
@@ -97,6 +117,7 @@ def _per_token_group_quant_custom_op(
|
||||
sorted_token_ids: Optional[torch.Tensor] = None,
|
||||
num_tokens_post_padded: Optional[torch.Tensor] = None,
|
||||
num_valid_tokens: int = 0,
|
||||
silu_and_mul_mode: str = "legacy",
|
||||
) -> None:
|
||||
num_groups = output_q.shape[-1] // group_size
|
||||
row_major, aligned = _infer_scale_layout(output_s, scale_ue8m0, num_groups)
|
||||
@@ -111,6 +132,7 @@ def _per_token_group_quant_custom_op(
|
||||
masked_m is not None,
|
||||
is_arch_support_pdl(),
|
||||
sorted_token_ids is not None,
|
||||
silu_and_mul_mode,
|
||||
)
|
||||
if masked_m is not None:
|
||||
module.per_token_group_quant(
|
||||
@@ -186,6 +208,7 @@ def per_token_group_quant(
|
||||
sorted_token_ids: Optional[torch.Tensor] = None,
|
||||
num_tokens_post_padded: Optional[torch.Tensor] = None,
|
||||
num_valid_tokens: int = 0,
|
||||
silu_and_mul_mode: Literal["legacy", "dense"] = "legacy",
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Per-token-group quantization. Returns ``(output_q, output_s)``.
|
||||
|
||||
@@ -207,6 +230,11 @@ def per_token_group_quant(
|
||||
int32 contiguous -> row-major UE8M0 bytes packed 4-per-int32
|
||||
The packed layouts require ``scale_ue8m0=True``.
|
||||
|
||||
With ``fuse_silu_and_mul=True``, ``silu_and_mul_mode="dense"`` matches the
|
||||
dense CUDA activation: FP32 SiLU and multiply, then one source-dtype round
|
||||
before quantizing. The default retains the existing early SiLU rounding.
|
||||
Dense mode supports flat BF16/FP16 inputs, FP8 output and groups of 128.
|
||||
|
||||
``expected_m`` (masked only) is an optional expected-tokens-per-expert hint.
|
||||
|
||||
``sorted_token_ids`` selects the sorted layout: ``input`` rows follow
|
||||
@@ -242,5 +270,6 @@ def per_token_group_quant(
|
||||
sorted_token_ids=sorted_token_ids,
|
||||
num_tokens_post_padded=num_tokens_post_padded,
|
||||
num_valid_tokens=num_valid_tokens,
|
||||
silu_and_mul_mode=silu_and_mul_mode,
|
||||
)
|
||||
return output_q, output_s
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Dense activation/quantization fusion for the block-FP8 DeepGEMM path."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.fused_op import get_fused_op_backend
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_forward,
|
||||
get_lora,
|
||||
get_parallel,
|
||||
get_platform,
|
||||
)
|
||||
from sglang.srt.utils import is_cuda
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.linear import RowParallelLinear
|
||||
|
||||
|
||||
class DenseSiluAndMulQuant:
|
||||
"""Keep the normal activation unless down_proj can consume our FP8 tuple."""
|
||||
|
||||
def __init__(self, down_proj: RowParallelLinear) -> None:
|
||||
self.enabled = False
|
||||
self.down_proj = down_proj
|
||||
self.quant_method = down_proj.quant_method
|
||||
if not (is_cuda() and get_platform().is_sm100 and get_parallel().tp_size == 1):
|
||||
return
|
||||
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.linear import RowParallelLinear
|
||||
from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod
|
||||
from sglang.srt.layers.quantization.fp8_utils import (
|
||||
_deepgemm_block_fp8_supported,
|
||||
deepgemm_w8a8_block_fp8_linear_with_fallback,
|
||||
)
|
||||
|
||||
method = self.quant_method
|
||||
if not (
|
||||
type(down_proj) is RowParallelLinear
|
||||
and type(method) is Fp8LinearMethod
|
||||
and method.weight_block_size == [128, 128]
|
||||
and method.quant_config.activation_scheme == "dynamic"
|
||||
and not (method.use_marlin or method.use_mxfp8 or method.block_fp8_as_mxfp8)
|
||||
and deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
|
||||
and down_proj.input_is_parallel
|
||||
):
|
||||
return
|
||||
backend = method.w8a8_block_fp8_linear
|
||||
if isinstance(backend, partial):
|
||||
if backend.args or set(backend.keywords) - {"act_scale_ue8m0"}:
|
||||
return
|
||||
backend = backend.func
|
||||
# The supported loader preserves matrix geometry, not tensor identity/device.
|
||||
self.weight_shape = (down_proj.output_size, down_proj.input_size_per_partition)
|
||||
self.enabled = (
|
||||
backend is deepgemm_w8a8_block_fp8_linear_with_fallback
|
||||
and _deepgemm_block_fp8_supported(
|
||||
self.weight_shape, method.weight_block_size, torch.bfloat16
|
||||
)
|
||||
)
|
||||
|
||||
def forward_decode(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
act_fn: SiluAndMul,
|
||||
down_proj: RowParallelLinear,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Try fusion for decode; callers must bypass this helper in other modes."""
|
||||
if not self.enabled or torch.compiler.is_compiling():
|
||||
return act_fn(x)
|
||||
|
||||
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
|
||||
deterministic = get_exec().deterministic
|
||||
if (
|
||||
down_proj is not self.down_proj
|
||||
or down_proj.quant_method is not self.quant_method
|
||||
or down_proj.weight.shape != self.weight_shape
|
||||
or type(act_fn) is not SiluAndMul
|
||||
or (act_fn._forward_method or act_fn._resolve_forward_method())
|
||||
!= act_fn.forward_cuda
|
||||
or act_fn.is_torch_compile
|
||||
or get_fused_op_backend() is not None
|
||||
or deterministic.enable_deterministic_inference
|
||||
or deterministic.rl_on_policy_target is not None
|
||||
or is_batch_invariant_mode_enabled()
|
||||
or get_lora().enable_lora
|
||||
or get_forward().sp_active
|
||||
or x.dtype != torch.bfloat16
|
||||
or not x.is_cuda
|
||||
or x.ndim != 2
|
||||
or x.shape[0] == 0
|
||||
or not x.is_contiguous()
|
||||
or x.data_ptr() % 32 != 0
|
||||
or x.stride(0) % 16 != 0
|
||||
or x.device != down_proj.weight.device
|
||||
or x.shape[1] != 2 * self.weight_shape[1]
|
||||
):
|
||||
return act_fn(x)
|
||||
|
||||
from sglang.kernels.ops.quantization.per_token_group_quant import (
|
||||
per_token_group_quant,
|
||||
)
|
||||
|
||||
return per_token_group_quant(
|
||||
x,
|
||||
group_size=128,
|
||||
scale_ue8m0=True,
|
||||
column_major_scales=True,
|
||||
fuse_silu_and_mul=True,
|
||||
silu_and_mul_mode="dense",
|
||||
)
|
||||
@@ -1453,6 +1453,25 @@ def _deepgemm_block_fp8_supported(
|
||||
return list(block_size) == [128, 128] and n % 64 == 0 and k % 128 == 0
|
||||
|
||||
|
||||
def _deepgemm_packed_input_valid(input, weight, input_scale, weight_scale) -> bool:
|
||||
"""Validate the group-128, column-major TMA layout for packed activations."""
|
||||
if input.ndim != 2 or weight.ndim != 2:
|
||||
return False
|
||||
m, k = input.shape
|
||||
packed_groups = ceil_div(k // 128, 4)
|
||||
padded_m = ceil_align(m, 4)
|
||||
return (
|
||||
input.is_cuda
|
||||
and input.is_contiguous()
|
||||
and k == weight.shape[1]
|
||||
and input.device == weight.device == input_scale.device == weight_scale.device
|
||||
and input_scale.shape == (m, packed_groups)
|
||||
and input_scale.stride() == (1, padded_m)
|
||||
and input.data_ptr() % 16 == 0
|
||||
and input_scale.data_ptr() % 16 == 0
|
||||
)
|
||||
|
||||
|
||||
def deepgemm_w8a8_block_fp8_linear_with_fallback(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
@@ -1463,18 +1482,36 @@ def deepgemm_w8a8_block_fp8_linear_with_fallback(
|
||||
act_scale_ue8m0: bool = False,
|
||||
weight_bf16: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
if input_scale is not None:
|
||||
# Block32 pre-quantized input reaches Triton below on UE8M0 DeepGEMM.
|
||||
assert (
|
||||
list(block_size) == [32, 32] or not deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
|
||||
)
|
||||
assert input.dtype == torch.float8_e4m3fn
|
||||
output_dtype = torch.bfloat16 if input_scale is not None else input.dtype
|
||||
|
||||
# TODO: https://github.com/sgl-project/sglang/pull/6890#issuecomment-2943395737
|
||||
if not _deepgemm_block_fp8_supported(
|
||||
deepgemm_supported = weight.ndim == 2 and _deepgemm_block_fp8_supported(
|
||||
weight.shape, block_size, output_dtype
|
||||
) or _ue8m0_deepgemm_skips_block32(block_size, weight_scale, input_scale):
|
||||
)
|
||||
if input_scale is not None:
|
||||
if input_scale.dtype == torch.int32:
|
||||
# Packed activation scales cannot go through the Triton fallback.
|
||||
if not (
|
||||
deepgemm_supported
|
||||
and deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
|
||||
and list(block_size) == [128, 128]
|
||||
and _deepgemm_packed_input_valid(
|
||||
input, weight, input_scale, weight_scale
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
"Pre-quantized DeepGEMM block-FP8 input requires supported "
|
||||
"[128, 128] weights and TMA-aligned, column-major packed UE8M0 scales"
|
||||
)
|
||||
else:
|
||||
# Block32 pre-quantized input uses the Triton fallback.
|
||||
assert (
|
||||
list(block_size) == [32, 32]
|
||||
or not deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
|
||||
)
|
||||
assert input.dtype == torch.float8_e4m3fn
|
||||
# TODO: https://github.com/sgl-project/sglang/pull/6890#issuecomment-2943395737
|
||||
if not deepgemm_supported or _ue8m0_deepgemm_skips_block32(
|
||||
block_size, weight_scale, input_scale
|
||||
):
|
||||
# fall back to triton
|
||||
# If weight_scale is in UE8M0 packed format (int32), convert back to float32
|
||||
# UE8M0 format has shape (N, K//block_k//4) with dtype int32
|
||||
@@ -1495,8 +1532,8 @@ def deepgemm_w8a8_block_fp8_linear_with_fallback(
|
||||
)
|
||||
|
||||
if input_scale is not None:
|
||||
# Pre-quantized activations retain their per-token group size and
|
||||
# fp32 scales. DeepGEMM aligns scale storage for TMA when needed.
|
||||
# Pre-quantized activations retain their group size and scale format.
|
||||
# DeepGEMM aligns fp32 scale storage for TMA when needed.
|
||||
input_2d = input.view(-1, input.shape[-1])
|
||||
output = w8a8_block_fp8_matmul_deepgemm(
|
||||
input_2d,
|
||||
|
||||
@@ -49,6 +49,7 @@ from sglang.srt.layers.linear import (
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
|
||||
from sglang.srt.layers.pooler import Pooler, PoolingType
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.fp8_activation import DenseSiluAndMulQuant
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
|
||||
@@ -118,6 +119,7 @@ class LlamaMLP(nn.Module):
|
||||
)
|
||||
self.act_fn = SiluAndMul()
|
||||
self.dual_gemm = DualGemm(self.gate_up_proj, self.down_proj, hidden_size)
|
||||
self.activation_quant = DenseSiluAndMulQuant(self.down_proj)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -128,7 +130,12 @@ class LlamaMLP(nn.Module):
|
||||
x = self.dual_gemm(x, self.gate_up_proj)
|
||||
else:
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
if forward_batch is not None and forward_batch.forward_mode.is_decode():
|
||||
x = self.activation_quant.forward_decode(
|
||||
gate_up, self.act_fn, self.down_proj
|
||||
)
|
||||
else:
|
||||
x = self.act_fn(gate_up)
|
||||
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
@@ -404,7 +411,7 @@ class LlamaDecoderLayer(nn.Module):
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
|
||||
@@ -46,6 +46,7 @@ from sglang.srt.layers.linear import (
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.pooler import Pooler, PoolingType
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.fp8_activation import DenseSiluAndMulQuant
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
|
||||
@@ -102,6 +103,7 @@ class Qwen2MLP(nn.Module):
|
||||
)
|
||||
self.act_fn = SiluAndMul()
|
||||
self.dual_gemm = DualGemm(self.gate_up_proj, self.down_proj, hidden_size)
|
||||
self.activation_quant = DenseSiluAndMulQuant(self.down_proj)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -115,7 +117,12 @@ class Qwen2MLP(nn.Module):
|
||||
x = self.dual_gemm(x, self.gate_up_proj)
|
||||
else:
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
if forward_batch is not None and forward_batch.forward_mode.is_decode():
|
||||
x = self.activation_quant.forward_decode(
|
||||
gate_up, self.act_fn, self.down_proj
|
||||
)
|
||||
else:
|
||||
x = self.act_fn(gate_up)
|
||||
x, _ = self.down_proj(x, forward_batch=forward_batch)
|
||||
return x
|
||||
|
||||
@@ -339,7 +346,7 @@ class Qwen2DecoderLayer(nn.Module):
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user