[Perf] mlp: Fuse dense SwiGLU with block-FP8 quantization on decode (#45219)

Co-authored-by: Jayden Yu <jaydenyu@fb.com>
Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
Jayden Yu
2026-10-23 07:49:39 +08:00
committed by GitHub
co-authored by Jayden Yu Xiaoyu Zhang
parent 5849ff5337
commit bb9e9d5632
11 changed files with 878 additions and 29 deletions
@@ -238,7 +238,8 @@ template <
bool kUe8m0_,
bool kRowMajor_,
bool kAligned_,
bool kFuseSiluAndMul_>
bool kFuseSiluAndMul_,
bool kDenseSiluAndMul = false>
struct QuantTrait {
// rename
using InputType = InputType_;
@@ -253,6 +254,7 @@ struct QuantTrait {
static constexpr uint32_t kNumLanes = kGroupSize / kVecSize;
static_assert(sizeof(InputType) == 2 || sizeof(InputType) == 4, "inputs must be 16-bit (bf16/fp16) or fp32");
static_assert(sizeof(InputType) == 2 || !kFuseSiluAndMul, "fp32 inputs do not implement the fused silu");
static_assert(!kDenseSiluAndMul || (kFuseSiluAndMul && kGroupSize == 128 && std::is_same_v<QuantType, fp8_e4m3_t>));
static_assert(16 <= kGroupSize && kGroupSize <= 256, "supported group sizes are 16..256");
static_assert(kGroupSize % kVecSize == 0 && 1 <= kNumLanes && kNumLanes <= device::kWarpThreads);
static_assert(!kUe8m0 || std::is_same_v<QuantType, fp8_e4m3_t>, "ue8m0 scales imply fp8 output");
@@ -302,8 +304,15 @@ struct QuantTrait {
#pragma unroll
for (uint32_t i = 0; i < kVecSize / 2; ++i) {
const auto gate = cast<float2>(in[i]);
const auto act = cast<T2>(float2{detail::silu(gate.x), detail::silu(gate.y)});
in[i] = __hmul2(act, up[i]);
if constexpr (kDenseSiluAndMul) {
// Match dense activation.cuh: round the product, not SiLU, to T.
const auto up_f32 = cast<float2>(up[i]);
in[i] = cast<T2>(
float2{(gate.x / (1.0f + expf(-gate.x))) * up_f32.x, (gate.y / (1.0f + expf(-gate.y))) * up_f32.y});
} else {
const auto act = cast<T2>(float2{detail::silu(gate.x), detail::silu(gate.y)});
in[i] = __hmul2(act, up[i]);
}
}
}
@@ -342,7 +351,9 @@ struct QuantTrait {
} else {
// fp32 scale: multiply in fp32 (hmul2 brings too much precision loss)
scale_inv = raw_scale;
const float quant_scale = kMaxValue / amax;
// Dense mode uses activation's compiler flags, but quantization must
// retain the fast division used by the standalone quantizer.
const float quant_scale = kDenseSiluAndMul ? __fdividef(kMaxValue, amax) : kMaxValue / amax;
const float2 quant_scale2 = {quant_scale, quant_scale};
#pragma unroll
for (uint32_t i = 0; i < kVecSize / 2; ++i) {
@@ -621,9 +632,11 @@ template <
bool kRowMajor,
bool kAligned,
bool kFuseSiluAndMul,
bool kUsePDL>
bool kUsePDL,
bool kDenseSiluAndMul = false>
struct PerTokenGroupQuantFlatKernel {
using Trait = QuantTrait<InputType, QuantType, kGroupSize, kUe8m0, kRowMajor, kAligned, kFuseSiluAndMul>;
using Trait =
QuantTrait<InputType, QuantType, kGroupSize, kUe8m0, kRowMajor, kAligned, kFuseSiluAndMul, kDenseSiluAndMul>;
static void run(tvm::ffi::TensorView input, tvm::ffi::TensorView output_q, tvm::ffi::TensorView output_s) {
const auto ctx = build_quant_context<Trait, /*kMasked=*/false>(input, output_q, output_s);
@@ -1,11 +1,12 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Optional, Tuple
from typing import TYPE_CHECKING, Literal, Optional, Tuple
import torch
from sglang.kernels.jit.utils import (
cache_once,
get_activation_cuda_cflags,
is_arch_support_pdl,
load_jit,
make_cpp_args,
@@ -33,10 +34,26 @@ def _jit_module(
masked_layout: bool,
use_pdl: bool,
sorted_layout: bool = False,
silu_and_mul_mode: str = "legacy",
) -> Module:
assert in_dtype in _SUPPORTED_INPUT_DTYPES
assert out_dtype in _SUPPORTED_OUTPUT_DTYPES
assert group_size in _SUPPORTED_GROUP_SIZES
if silu_and_mul_mode not in ("legacy", "dense"):
raise ValueError(f"Unknown silu_and_mul_mode: {silu_and_mul_mode}")
dense = silu_and_mul_mode == "dense"
if dense and not (
fuse_silu_and_mul
and in_dtype in (torch.bfloat16, torch.float16)
and out_dtype == torch.float8_e4m3fn
and group_size == 128
and not masked_layout
and not sorted_layout
):
raise ValueError(
"dense mode requires flat BF16/FP16 SiLU+mul, FP8 output and group_size=128"
)
# Only the flat dense variant adds a flag; legacy launcher arguments stay intact.
trait_args = make_cpp_args(
in_dtype,
out_dtype,
@@ -46,6 +63,7 @@ def _jit_module(
aligned,
fuse_silu_and_mul,
use_pdl,
*([True] if dense else []),
)
assert not (masked_layout and sorted_layout)
layout = "masked" if masked_layout else ("sorted" if sorted_layout else "flat")
@@ -56,7 +74,9 @@ def _jit_module(
layout,
cuda_files=["gemm/per_token_group_quant.cuh"],
cuda_wrappers=[("per_token_group_quant", f"{launcher}<{trait_args}>::run")],
extra_cuda_cflags=["--use_fast_math"],
extra_cuda_cflags=get_activation_cuda_cflags()
if dense
else ["--use_fast_math"],
)
@@ -97,6 +117,7 @@ def _per_token_group_quant_custom_op(
sorted_token_ids: Optional[torch.Tensor] = None,
num_tokens_post_padded: Optional[torch.Tensor] = None,
num_valid_tokens: int = 0,
silu_and_mul_mode: str = "legacy",
) -> None:
num_groups = output_q.shape[-1] // group_size
row_major, aligned = _infer_scale_layout(output_s, scale_ue8m0, num_groups)
@@ -111,6 +132,7 @@ def _per_token_group_quant_custom_op(
masked_m is not None,
is_arch_support_pdl(),
sorted_token_ids is not None,
silu_and_mul_mode,
)
if masked_m is not None:
module.per_token_group_quant(
@@ -186,6 +208,7 @@ def per_token_group_quant(
sorted_token_ids: Optional[torch.Tensor] = None,
num_tokens_post_padded: Optional[torch.Tensor] = None,
num_valid_tokens: int = 0,
silu_and_mul_mode: Literal["legacy", "dense"] = "legacy",
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Per-token-group quantization. Returns ``(output_q, output_s)``.
@@ -207,6 +230,11 @@ def per_token_group_quant(
int32 contiguous -> row-major UE8M0 bytes packed 4-per-int32
The packed layouts require ``scale_ue8m0=True``.
With ``fuse_silu_and_mul=True``, ``silu_and_mul_mode="dense"`` matches the
dense CUDA activation: FP32 SiLU and multiply, then one source-dtype round
before quantizing. The default retains the existing early SiLU rounding.
Dense mode supports flat BF16/FP16 inputs, FP8 output and groups of 128.
``expected_m`` (masked only) is an optional expected-tokens-per-expert hint.
``sorted_token_ids`` selects the sorted layout: ``input`` rows follow
@@ -242,5 +270,6 @@ def per_token_group_quant(
sorted_token_ids=sorted_token_ids,
num_tokens_post_padded=num_tokens_post_padded,
num_valid_tokens=num_valid_tokens,
silu_and_mul_mode=silu_and_mul_mode,
)
return output_q, output_s
@@ -0,0 +1,119 @@
"""Dense activation/quantization fusion for the block-FP8 DeepGEMM path."""
from __future__ import annotations
from functools import partial
from typing import TYPE_CHECKING
import torch
from sglang.kernels.fused_op import get_fused_op_backend
from sglang.srt.runtime_context import (
get_exec,
get_forward,
get_lora,
get_parallel,
get_platform,
)
from sglang.srt.utils import is_cuda
if TYPE_CHECKING:
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.linear import RowParallelLinear
class DenseSiluAndMulQuant:
"""Keep the normal activation unless down_proj can consume our FP8 tuple."""
def __init__(self, down_proj: RowParallelLinear) -> None:
self.enabled = False
self.down_proj = down_proj
self.quant_method = down_proj.quant_method
if not (is_cuda() and get_platform().is_sm100 and get_parallel().tp_size == 1):
return
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.linear import RowParallelLinear
from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod
from sglang.srt.layers.quantization.fp8_utils import (
_deepgemm_block_fp8_supported,
deepgemm_w8a8_block_fp8_linear_with_fallback,
)
method = self.quant_method
if not (
type(down_proj) is RowParallelLinear
and type(method) is Fp8LinearMethod
and method.weight_block_size == [128, 128]
and method.quant_config.activation_scheme == "dynamic"
and not (method.use_marlin or method.use_mxfp8 or method.block_fp8_as_mxfp8)
and deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
and down_proj.input_is_parallel
):
return
backend = method.w8a8_block_fp8_linear
if isinstance(backend, partial):
if backend.args or set(backend.keywords) - {"act_scale_ue8m0"}:
return
backend = backend.func
# The supported loader preserves matrix geometry, not tensor identity/device.
self.weight_shape = (down_proj.output_size, down_proj.input_size_per_partition)
self.enabled = (
backend is deepgemm_w8a8_block_fp8_linear_with_fallback
and _deepgemm_block_fp8_supported(
self.weight_shape, method.weight_block_size, torch.bfloat16
)
)
def forward_decode(
self,
x: torch.Tensor,
act_fn: SiluAndMul,
down_proj: RowParallelLinear,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""Try fusion for decode; callers must bypass this helper in other modes."""
if not self.enabled or torch.compiler.is_compiling():
return act_fn(x)
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
from sglang.srt.layers.activation import SiluAndMul
deterministic = get_exec().deterministic
if (
down_proj is not self.down_proj
or down_proj.quant_method is not self.quant_method
or down_proj.weight.shape != self.weight_shape
or type(act_fn) is not SiluAndMul
or (act_fn._forward_method or act_fn._resolve_forward_method())
!= act_fn.forward_cuda
or act_fn.is_torch_compile
or get_fused_op_backend() is not None
or deterministic.enable_deterministic_inference
or deterministic.rl_on_policy_target is not None
or is_batch_invariant_mode_enabled()
or get_lora().enable_lora
or get_forward().sp_active
or x.dtype != torch.bfloat16
or not x.is_cuda
or x.ndim != 2
or x.shape[0] == 0
or not x.is_contiguous()
or x.data_ptr() % 32 != 0
or x.stride(0) % 16 != 0
or x.device != down_proj.weight.device
or x.shape[1] != 2 * self.weight_shape[1]
):
return act_fn(x)
from sglang.kernels.ops.quantization.per_token_group_quant import (
per_token_group_quant,
)
return per_token_group_quant(
x,
group_size=128,
scale_ue8m0=True,
column_major_scales=True,
fuse_silu_and_mul=True,
silu_and_mul_mode="dense",
)
@@ -1453,6 +1453,25 @@ def _deepgemm_block_fp8_supported(
return list(block_size) == [128, 128] and n % 64 == 0 and k % 128 == 0
def _deepgemm_packed_input_valid(input, weight, input_scale, weight_scale) -> bool:
"""Validate the group-128, column-major TMA layout for packed activations."""
if input.ndim != 2 or weight.ndim != 2:
return False
m, k = input.shape
packed_groups = ceil_div(k // 128, 4)
padded_m = ceil_align(m, 4)
return (
input.is_cuda
and input.is_contiguous()
and k == weight.shape[1]
and input.device == weight.device == input_scale.device == weight_scale.device
and input_scale.shape == (m, packed_groups)
and input_scale.stride() == (1, padded_m)
and input.data_ptr() % 16 == 0
and input_scale.data_ptr() % 16 == 0
)
def deepgemm_w8a8_block_fp8_linear_with_fallback(
input: torch.Tensor,
weight: torch.Tensor,
@@ -1463,18 +1482,36 @@ def deepgemm_w8a8_block_fp8_linear_with_fallback(
act_scale_ue8m0: bool = False,
weight_bf16: Optional[torch.Tensor] = None,
) -> torch.Tensor:
if input_scale is not None:
# Block32 pre-quantized input reaches Triton below on UE8M0 DeepGEMM.
assert (
list(block_size) == [32, 32] or not deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
)
assert input.dtype == torch.float8_e4m3fn
output_dtype = torch.bfloat16 if input_scale is not None else input.dtype
# TODO: https://github.com/sgl-project/sglang/pull/6890#issuecomment-2943395737
if not _deepgemm_block_fp8_supported(
deepgemm_supported = weight.ndim == 2 and _deepgemm_block_fp8_supported(
weight.shape, block_size, output_dtype
) or _ue8m0_deepgemm_skips_block32(block_size, weight_scale, input_scale):
)
if input_scale is not None:
if input_scale.dtype == torch.int32:
# Packed activation scales cannot go through the Triton fallback.
if not (
deepgemm_supported
and deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
and list(block_size) == [128, 128]
and _deepgemm_packed_input_valid(
input, weight, input_scale, weight_scale
)
):
raise ValueError(
"Pre-quantized DeepGEMM block-FP8 input requires supported "
"[128, 128] weights and TMA-aligned, column-major packed UE8M0 scales"
)
else:
# Block32 pre-quantized input uses the Triton fallback.
assert (
list(block_size) == [32, 32]
or not deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
)
assert input.dtype == torch.float8_e4m3fn
# TODO: https://github.com/sgl-project/sglang/pull/6890#issuecomment-2943395737
if not deepgemm_supported or _ue8m0_deepgemm_skips_block32(
block_size, weight_scale, input_scale
):
# fall back to triton
# If weight_scale is in UE8M0 packed format (int32), convert back to float32
# UE8M0 format has shape (N, K//block_k//4) with dtype int32
@@ -1495,8 +1532,8 @@ def deepgemm_w8a8_block_fp8_linear_with_fallback(
)
if input_scale is not None:
# Pre-quantized activations retain their per-token group size and
# fp32 scales. DeepGEMM aligns scale storage for TMA when needed.
# Pre-quantized activations retain their group size and scale format.
# DeepGEMM aligns fp32 scale storage for TMA when needed.
input_2d = input.view(-1, input.shape[-1])
output = w8a8_block_fp8_matmul_deepgemm(
input_2d,
+9 -2
View File
@@ -49,6 +49,7 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
from sglang.srt.layers.pooler import Pooler, PoolingType
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.fp8_activation import DenseSiluAndMulQuant
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
@@ -118,6 +119,7 @@ class LlamaMLP(nn.Module):
)
self.act_fn = SiluAndMul()
self.dual_gemm = DualGemm(self.gate_up_proj, self.down_proj, hidden_size)
self.activation_quant = DenseSiluAndMulQuant(self.down_proj)
def forward(
self,
@@ -128,7 +130,12 @@ class LlamaMLP(nn.Module):
x = self.dual_gemm(x, self.gate_up_proj)
else:
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
if forward_batch is not None and forward_batch.forward_mode.is_decode():
x = self.activation_quant.forward_decode(
gate_up, self.act_fn, self.down_proj
)
else:
x = self.act_fn(gate_up)
x, _ = self.down_proj(x)
return x
@@ -404,7 +411,7 @@ class LlamaDecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
hidden_states = self.mlp(hidden_states)
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
return self.ffn_boundary.finish(hidden_states, forward_batch)
+9 -2
View File
@@ -46,6 +46,7 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.pooler import Pooler, PoolingType
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.fp8_activation import DenseSiluAndMulQuant
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
@@ -102,6 +103,7 @@ class Qwen2MLP(nn.Module):
)
self.act_fn = SiluAndMul()
self.dual_gemm = DualGemm(self.gate_up_proj, self.down_proj, hidden_size)
self.activation_quant = DenseSiluAndMulQuant(self.down_proj)
def forward(
self,
@@ -115,7 +117,12 @@ class Qwen2MLP(nn.Module):
x = self.dual_gemm(x, self.gate_up_proj)
else:
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
if forward_batch is not None and forward_batch.forward_mode.is_decode():
x = self.activation_quant.forward_decode(
gate_up, self.act_fn, self.down_proj
)
else:
x = self.act_fn(gate_up)
x, _ = self.down_proj(x, forward_batch=forward_batch)
return x
@@ -339,7 +346,7 @@ class Qwen2DecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
hidden_states = self.mlp(hidden_states)
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
return self.ffn_boundary.finish(hidden_states, forward_batch)
@@ -0,0 +1,147 @@
"""Paired B200 MLP timing for dense SwiGLU fusion.
Synthetic weights test the operator pipeline, not model quality or serving TBT.
Operator parity and the opt-in numerical sweep live in test_per_token_group_quant.py.
"""
import argparse
import json
import random
import statistics
import subprocess
import torch
from sglang.kernels.jit.benchmark import marker
from sglang.kernels.ops.activation.activation import silu_and_mul
from sglang.kernels.ops.quantization.per_token_group_quant import per_token_group_quant
def mlp(x, gate_up, gate_scale, down, down_scale, *, fused):
from sglang.srt.layers.quantization.fp8_utils import (
deepgemm_w8a8_block_fp8_linear_with_fallback as linear,
)
projected = linear(x, gate_up, [128, 128], gate_scale)
if fused:
q, s = per_token_group_quant(
projected,
scale_ue8m0=True,
column_major_scales=True,
fuse_silu_and_mul=True,
silu_and_mul_mode="dense",
)
return linear(q, down, [128, 128], down_scale, s)
return linear(silu_and_mul(projected), down, [128, 128], down_scale)
def benchmark(args):
from functools import partial
torch.manual_seed(20261017)
order = random.Random(17)
for m in args.batches:
k, h = args.intermediate_size, 4096
assert k % 512 == 0
x = torch.randn(m, h, device="cuda", dtype=torch.bfloat16)
up = (torch.randn(2 * k, h, device="cuda") * 0.01).to(torch.float8_e4m3fn)
down = (torch.randn(h, k, device="cuda") * 0.01).to(torch.float8_e4m3fn)
# Packed exponent 127 represents scale=1, using DeepGEMM's weight layout.
us = torch.full(
(h // 512, 2 * k), 0x7F7F7F7F, device="cuda", dtype=torch.int32
).T
ds = torch.full((k // 512, h), 0x7F7F7F7F, device="cuda", dtype=torch.int32).T
inputs = (x, up, us, down, ds)
variants = {
"eager": partial(mlp, fused=False),
"fused": partial(mlp, fused=True),
"control": partial(mlp, fused=False),
}
torch.testing.assert_close(
variants["eager"](*inputs), variants["fused"](*inputs), rtol=0, atol=0
)
if args.profile:
for name in ("eager", "fused"):
with torch.profiler.profile(
activities=[
torch.profiler.ProfilerActivity.CPU,
torch.profiler.ProfilerActivity.CUDA,
]
) as prof:
variants[name](*inputs)
torch.cuda.synchronize()
kernels = [
e.name
for e in prof.events()
if e.device_type == torch.autograd.DeviceType.CUDA
]
print(
json.dumps({"batch": m, "variant": name, "CUDA_events": kernels}),
flush=True,
)
samples = {name: [] for name in variants}
for _ in range(args.rounds):
names = list(variants)
order.shuffle(names)
for name in names:
# Rotate all inputs, including both GEMM weights, beyond L2.
result = marker.do_bench(
variants[name],
input_args=inputs,
metrics=(0.5,),
estimated_time_ms=30,
warmup_iters=3,
graph_clone_args="all",
disable_log_bandwidth=True,
)
samples[name].append(result.times[0] * 1e6)
eager, fused, control = (samples[n] for n in ("eager", "fused", "control"))
print(
json.dumps(
{
"batch": m,
"intermediate_size": k,
"rounds": args.rounds,
"cold_L2": True,
"raw_us": samples,
"median_us": {n: statistics.median(s) for n, s in samples.items()},
"paired_wins": sum(a > b for a, b in zip(eager, fused)),
"paired_saving_us": statistics.median(
a - b for a, b in zip(eager, fused)
),
"control_adjusted_saving_us": statistics.median(
(a + c) / 2 - b for a, b, c in zip(eager, fused, control)
),
"AA_median_abs_difference_us": statistics.median(
abs(a - c) for a, c in zip(eager, control)
),
}
),
flush=True,
)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--rounds", type=int, default=10)
parser.add_argument("--batches", type=int, nargs="+", default=[1, 32, 128])
parser.add_argument("--intermediate-size", type=int, default=14336)
parser.add_argument("--profile", action="store_true")
args = parser.parse_args()
print(
json.dumps(
{
"base_commit": subprocess.check_output(
["git", "rev-parse", "HEAD"], text=True
).strip(),
"worktree_dirty": bool(
subprocess.check_output(["git", "status", "--porcelain"], text=True)
),
"torch": torch.__version__,
"gpu": torch.cuda.get_device_name(),
}
),
flush=True,
)
with torch.inference_mode():
benchmark(args)
@@ -1,5 +1,6 @@
from sglang.kernels.jit.benchmark import marker
from sglang.kernels.jit.benchmark.utils import create_empty, create_random
from sglang.kernels.ops.activation.activation import silu_and_mul
from sglang.kernels.ops.quantization.fp8_kernel import (
create_per_token_group_quant_fp8_output_scale,
fp8_dtype,
@@ -18,7 +19,7 @@ from sglang.kernels.ops.quantization.per_token_group_quant_8bit_v2 import (
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(
est_time=25, stage="base-b-kernel-benchmark", runner_config="1-gpu-large"
est_time=35, stage="base-b-kernel-benchmark", runner_config="1-gpu-large"
)
HIDDEN = 2048
@@ -73,5 +74,23 @@ def benchmark(group_size: int, layout: str, num_tokens: int, impl: str):
)
def _dense_activation_quant(x, fused):
return per_token_group_quant(
x if fused else silu_and_mul(x),
fuse_silu_and_mul=fused,
silu_and_mul_mode="dense" if fused else "legacy",
scale_ue8m0=True,
column_major_scales=True,
)
@marker.parametrize("num_tokens", [1, 32, 128], ci_vals=[1, 32])
@marker.benchmark("fused", [False, True])
def benchmark_dense(num_tokens: int, fused: bool):
x = create_random(num_tokens, 8192)
return marker.do_bench(_dense_activation_quant, input_args=(x, fused))
if __name__ == "__main__":
benchmark.run()
benchmark_dense.run()
@@ -21,7 +21,8 @@ import itertools
import pytest
import torch
from sglang.kernels.jit.utils import get_ci_test_range
from sglang.kernels.jit.utils import get_ci_test_range, should_run_full_tests
from sglang.kernels.ops.activation.activation import silu_and_mul
from sglang.kernels.ops.quantization.fp8_kernel import (
create_per_token_group_quant_fp8_output_scale,
fp8_dtype,
@@ -32,9 +33,9 @@ from sglang.kernels.ops.quantization.per_token_group_quant import (
)
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=65, stage="base-b-kernel-unit", runner_config="1-gpu-large")
register_cuda_ci(est_time=90, stage="base-b-kernel-unit", runner_config="4-gpu-b200")
register_cuda_ci(est_time=120, stage="nightly", runner_config="1-gpu-large")
register_cuda_ci(est_time=120, stage="base-b-kernel-unit", runner_config="1-gpu-large")
register_cuda_ci(est_time=200, stage="base-b-kernel-unit", runner_config="4-gpu-b200")
register_cuda_ci(est_time=180, stage="nightly", runner_config="1-gpu-large")
G = 128
FMAX = float(fp8_max) # 448 for e4m3
@@ -527,6 +528,124 @@ def test_fused_silu(scale_ue8m0, column_major):
assert _dequant_rel_err(x_q, deq_scale, act, G) < 0.05
def _check_dense_silu(x, scale_ue8m0, column_major):
original = x.view(torch.int16).clone()
kwargs = dict(scale_ue8m0=scale_ue8m0, column_major_scales=column_major)
expected_q, expected_s = per_token_group_quant(silu_and_mul(x), **kwargs)
q, s = per_token_group_quant(
x, fuse_silu_and_mul=True, silu_and_mul_mode="dense", **kwargs
)
assert torch.equal(q.view(torch.uint8), expected_q.view(torch.uint8))
assert torch.equal(s.view(torch.int32), expected_s.view(torch.int32))
assert torch.equal(x.view(torch.int16), original)
@pytest.mark.parametrize(
"dtype,scale_ue8m0,column_major",
get_ci_test_range(
list(
itertools.product(
[torch.bfloat16, torch.float16], [False, True], [False, True]
)
),
[
(torch.bfloat16, True, True),
(torch.float16, True, True),
(torch.bfloat16, False, False),
],
),
)
@pytest.mark.parametrize(
"shape",
get_ci_test_range(
[(1, 768), (3, 4096), (32, 12288), (32, 14336), (128, 28672)],
[(1, 768), (32, 14336)],
),
)
def test_dense_silu_bitexact(dtype, shape, scale_ue8m0, column_major):
"""Dense fusion preserves the separately stored activation's rounding.
The legacy fused mode rounds SiLU before multiplication and fails this
comparison. Use the real CUDA activation as the compiler-math reference.
"""
torch.manual_seed(42)
m, h = shape
x = torch.randn(m, 2 * h, device="cuda", dtype=dtype) * 3
x[0, :8] = torch.tensor(
[0.0, -0.0, 1e-6, -1e-6, 100, -100, float("inf"), float("nan")],
device="cuda",
dtype=dtype,
)
_check_dense_silu(x, scale_ue8m0, column_major)
@pytest.mark.skipif(
not should_run_full_tests(),
reason="Set SGLANG_JIT_KERNEL_RUN_FULL_TESTS=1 for the sweep",
)
def test_dense_silu_extended():
"""Opt-in amplitude/layout sweep and all 65,536 BF16/FP16 gate encodings."""
torch.manual_seed(20261017)
layouts = list(itertools.product((False, True), repeat=2))
for dtype in (torch.bfloat16, torch.float16):
for (m, k), amplitude in itertools.product(
((1, 768), (3, 4096), (32, 12288), (32, 14336), (128, 28672)),
(0, 0.001, 1, 10, 100),
):
x = (torch.randn(m, 2 * k, device="cuda") * amplitude).to(dtype)
for ue8m0, column in layouts:
_check_dense_silu(x, ue8m0, column)
# Signed zeros, subnormals, finite extremes, infinities and NaNs.
gate = (
torch.arange(65536, device="cuda", dtype=torch.int32)
.to(torch.int16)
.view(dtype)
.reshape(64, 1024)
)
for up_value in (0, 0.25, 1, 1.125, -2.25):
x = torch.cat((gate, torch.full_like(gate, up_value)), dim=1)
for ue8m0, column in layouts:
_check_dense_silu(x, ue8m0, column)
def test_dense_silu_graph_replay():
"""Replays must use new input values and retain the allocated output buffers."""
x = torch.randn(3, 1536, device="cuda", dtype=torch.bfloat16)
kwargs = dict(scale_ue8m0=True, column_major_scales=True)
q, s = per_token_group_quant(
x, fuse_silu_and_mul=True, silu_and_mul_mode="dense", **kwargs
)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
per_token_group_quant(
x, q, s, fuse_silu_and_mul=True, silu_and_mul_mode="dense", **kwargs
)
for amplitude in (0, 0.001, 10):
x.normal_().mul_(amplitude)
graph.replay()
expected_q, expected_s = per_token_group_quant(silu_and_mul(x), **kwargs)
assert torch.equal(q.view(torch.uint8), expected_q.view(torch.uint8))
assert torch.equal(s, expected_s)
@pytest.mark.parametrize(
"overrides",
[
{"silu_and_mul_mode": "unknown"},
{"fuse_silu_and_mul": False},
{"out_dtype": torch.int8},
{"group_size": 64},
{"masked_m": torch.tensor([1], dtype=torch.int32)},
],
)
def test_dense_silu_invalid_mode(overrides):
x = torch.zeros(1, 256, device="cuda", dtype=torch.bfloat16)
kwargs = dict(fuse_silu_and_mul=True, silu_and_mul_mode="dense")
kwargs.update(overrides)
with pytest.raises(ValueError, match="mode"):
per_token_group_quant(x, **kwargs)
# --------------------------------------------------------------------------- #
# Masked EP-MoE schedule.
# --------------------------------------------------------------------------- #
@@ -0,0 +1,312 @@
"""Dense SwiGLU/block-FP8 integration: real MLPs, dispatch and graph replay."""
import unittest
from types import SimpleNamespace
from unittest import mock
import torch
from sglang.srt.layers.quantization import fp8_utils
from sglang.srt.layers.quantization.fp8 import Fp8Config
from sglang.srt.layers.quantization.fp8_utils import Fp8GemmRunnerBackend
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.model_loader.utils import set_default_torch_dtype
from sglang.srt.utils import get_device_sm
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.layer_ut_utils import init_single_process_dist
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=45, stage="base-b", runner_config="4-gpu-b200")
@unittest.skipUnless(get_device_sm() in (100, 103), "Dense block-FP8 fusion on SM100")
@mock.patch.object(fp8_utils, "FP8_GEMM_RUNNER_BACKEND", Fp8GemmRunnerBackend.DEEP_GEMM)
class TestDenseSiluAndMulQuant(CustomTestCase):
@classmethod
def setUpClass(cls):
init_single_process_dist()
def _make_mlp(self, model):
from sglang.srt.models.llama import LlamaMLP
from sglang.srt.models.qwen2 import Qwen2MLP
config = Fp8Config(
is_checkpoint_fp8_serialized=True,
activation_scheme="dynamic",
weight_block_size=[128, 128],
)
with set_default_torch_dtype(torch.bfloat16):
mlp = {"llama": LlamaMLP, "qwen": Qwen2MLP}[model](
512,
768,
"silu",
quant_config=config,
reduce_results=False,
).cuda()
with torch.no_grad():
for layer in (mlp.gate_up_proj, mlp.down_proj):
weight = (
torch.randn(layer.weight.shape, device="cuda", dtype=torch.bfloat16)
* 0.02
)
layer.weight.copy_(weight.to(torch.float8_e4m3fn))
layer.weight_scale_inv.fill_(1)
layer.quant_method.process_weights_after_loading(layer)
return mlp
def test_mlp_parity_and_graph_replay(self):
"""Cold first-decode dispatch and graph replay must match the unfused MLP."""
batch = SimpleNamespace(forward_mode=ForwardMode.DECODE)
torch.manual_seed(123)
with torch.inference_mode():
for model in ("llama", "qwen"):
mlp = self._make_mlp(model)
self.assertTrue(mlp.activation_quant.enabled)
self.assertIsNone(mlp.dual_gemm.mode)
self.assertIsNone(mlp.act_fn._forward_method)
for m in (1, 3, 32, 128):
with self.subTest(model=model, m=m):
x = torch.randn(m, 512, device="cuda", dtype=torch.bfloat16)
gate_up, _ = mlp.gate_up_proj(x)
quantized = mlp.activation_quant.forward_decode(
gate_up, mlp.act_fn, mlp.down_proj
)
self.assertIsInstance(quantized, tuple)
expected = mlp(x)
actual = mlp(x, batch)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
actual = mlp(x, batch)
for amplitude in (0, 0.1, 3):
x.normal_().mul_(amplitude)
graph.replay()
torch.testing.assert_close(actual, mlp(x), rtol=0, atol=0)
@torch.inference_mode()
def test_non_decode_bypass(self):
"""Every non-decode mode must avoid the helper and preserve MLP output."""
for model in ("llama", "qwen"):
mlp = self._make_mlp(model)
x = torch.randn(1, 512, device="cuda", dtype=torch.bfloat16)
expected = mlp(x)
with mock.patch.object(
type(mlp.activation_quant),
"forward_decode",
side_effect=AssertionError("non-decode entered fusion dispatch"),
):
for mode in (None, *ForwardMode):
if mode == ForwardMode.DECODE:
continue
with self.subTest(model=model, mode=mode):
batch = (
SimpleNamespace(forward_mode=mode)
if mode is not None
else None
)
torch.testing.assert_close(
mlp(x, batch), expected, rtol=0, atol=0
)
@torch.inference_mode()
def test_decoder_handoff(self):
"""Dropping forward_batch between the decoder and real MLP must fail."""
from sglang.srt.models.llama import LlamaDecoderLayer
from sglang.srt.models.qwen2 import Qwen2DecoderLayer
batch = SimpleNamespace(forward_mode=ForwardMode.DECODE)
boundary = SimpleNamespace(
prepare=lambda x, batch, **kw: x, finish=lambda x, batch: x
)
for model, decoder_cls in (
("llama", LlamaDecoderLayer),
("qwen", Qwen2DecoderLayer),
):
mlp = self._make_mlp(model)
x = torch.randn(1, 512, device="cuda", dtype=torch.bfloat16)
expected = mlp(x)
# Only attention/communication are stubbed; run the real decoder and MLP.
decoder = SimpleNamespace(
attn_boundary=boundary,
ffn_boundary=boundary,
self_attn=lambda **kw: kw["hidden_states"],
mlp=mlp,
)
with (
self.subTest(model=model),
mock.patch.object(
mlp.act_fn,
"forward",
side_effect=AssertionError("decode used separate activation"),
),
):
actual = decoder_cls.forward(decoder, None, x, batch)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
def test_decode_fallback_overrides_and_inputs(self):
"""Overrides and unsupported inputs must retain the original activation."""
from sglang.kernels.fused_op import get_fused_op_backend, set_fused_op_backend
from sglang.kernels.spec import KernelBackend
from sglang.srt.runtime_context import get_exec, get_forward, get_lora
mlp = self._make_mlp("llama")
x = torch.randn(3, 1536, device="cuda", dtype=torch.bfloat16)
def check(*, value=x, projection=mlp.down_proj):
actual = mlp.activation_quant.forward_decode(value, mlp.act_fn, projection)
self.assertIsInstance(actual, torch.Tensor)
torch.testing.assert_close(actual, mlp.act_fn(value), rtol=0, atol=0)
with get_forward().scoped(sp_active=True):
check()
with get_lora().override(enable_lora=True):
check()
with get_exec().deterministic.override(enable_deterministic_inference=True):
check()
with mock.patch.object(
mlp.act_fn, "_forward_method", mlp.act_fn.forward_native
):
check()
old_backend = get_fused_op_backend()
try:
set_fused_op_backend(KernelBackend.TORCH)
check()
finally:
set_fused_op_backend(old_backend)
for dtype in (torch.float16, torch.float32):
check(value=x.to(dtype))
# Contiguous single-row views may still have an unaligned row stride.
check(value=x[:1].as_strided((1, 1536), (1, 1)))
wrapped_projection = SimpleNamespace(
quant_method=mlp.down_proj.quant_method,
weight=mlp.down_proj.weight,
)
check(projection=wrapped_projection)
actual = mlp.activation_quant.forward_decode(
x, torch.nn.Identity(), mlp.down_proj
)
self.assertIs(actual, x)
mlp.act_fn.enter_torch_compile(x.shape[0])
try:
check()
compiled = torch.compile(
lambda value: mlp.activation_quant.forward_decode(
value, mlp.act_fn, mlp.down_proj
),
backend="eager",
fullgraph=True,
)
torch.testing.assert_close(
compiled(x), mlp.act_fn.forward_native(x), rtol=0, atol=0
)
finally:
mlp.act_fn.leave_torch_compile()
def test_constructor_gates(self):
"""Other quantization/backends must retain activation tensors for down_proj."""
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.linear import RowParallelLinear
from sglang.srt.layers.quantization.fp8_activation import DenseSiluAndMulQuant
from sglang.srt.runtime_context import override_platform
from sglang.test.parallel_groups import parallel_scope
cases = [
(None, "deep_gemm"),
({}, "deep_gemm"),
({"weight_block_size": [128, 128]}, "triton"),
({"use_mxfp8": True}, "auto"),
({"weight_block_size": [32, 32], "scale_fmt": "ue8m0"}, "auto"),
]
act = SiluAndMul()
x = torch.randn(3, 1536, device="cuda", dtype=torch.bfloat16)
expected = act(x)
for options, backend in cases:
config = (
Fp8Config(is_checkpoint_fp8_serialized=True, **options)
if options is not None
else None
)
with (
self.subTest(config=options, backend=backend),
mock.patch.object(
fp8_utils, "FP8_GEMM_RUNNER_BACKEND", Fp8GemmRunnerBackend(backend)
),
):
layer = RowParallelLinear(
768,
512,
bias=False,
params_dtype=torch.bfloat16,
quant_config=config,
).cuda()
helper = DenseSiluAndMulQuant(layer)
torch.testing.assert_close(
helper.forward_decode(x, act, layer), expected, rtol=0, atol=0
)
mlp = self._make_mlp("llama")
for scope in (
parallel_scope(tp_size=2, attn_tp_size=2, moe_tp_size=2),
override_platform(is_sm100=False),
):
with scope:
helper = DenseSiluAndMulQuant(mlp.down_proj)
torch.testing.assert_close(
helper.forward_decode(x, act, mlp.down_proj), expected, rtol=0, atol=0
)
@torch.inference_mode()
def test_fixed_geometry_and_live_projection_guards(self):
"""Cache support rules, but reject changed geometry/methods and other layers."""
from sglang.srt.layers.linear import RowParallelLinear
from sglang.srt.layers.quantization.fp8_activation import DenseSiluAndMulQuant
mlp = self._make_mlp("llama")
layer = mlp.down_proj
helper = mlp.activation_quant
x = torch.randn(3, 1536, device="cuda", dtype=torch.bfloat16)
expected = mlp.act_fn(x)
# Same-shape weight reloads remain eligible without redoing support checks.
with (
mock.patch.object(
layer, "weight", torch.nn.Parameter(layer.weight.clone(), False)
),
mock.patch.object(
fp8_utils, "_deepgemm_block_fp8_supported", side_effect=AssertionError
),
):
self.assertIsInstance(helper.forward_decode(x, mlp.act_fn, layer), tuple)
for attr, value in (
("weight", torch.nn.Parameter(layer.weight.reshape(1024, 384), False)),
("quant_method", object()),
):
with self.subTest(attr=attr), mock.patch.object(layer, attr, value):
torch.testing.assert_close(
helper.forward_decode(x, mlp.act_fn, layer),
expected,
rtol=0,
atol=0,
)
for n, k in ((512, 768), (513, 768), (512, 769)):
other = RowParallelLinear(
k,
n,
bias=False,
params_dtype=torch.bfloat16,
quant_config=layer.quant_method.quant_config,
)
with self.subTest(n=n, k=k):
self.assertEqual(
DenseSiluAndMulQuant(other).enabled, (n, k) == (512, 768)
)
torch.testing.assert_close(
helper.forward_decode(x, mlp.act_fn, other),
expected,
rtol=0,
atol=0,
)
if __name__ == "__main__":
unittest.main()
@@ -192,6 +192,46 @@ class TestFp8BlockwiseLinearBackends(_LinearBackendCheck):
def test_deep_gemm(self):
self._run("deep_gemm")
@unittest.skipUnless(
get_device_sm() in (100, 103), "Packed block-FP8 scales on SM100"
)
def test_deep_gemm_prequantized(self):
"""Packed activations must skip requantization and preserve the GEMM result."""
torch.manual_seed(9)
with mock.patch.object(
fp8_utils, "FP8_GEMM_RUNNER_BACKEND", Fp8GemmRunnerBackend.DEEP_GEMM
):
for m, n, k in FP8_BLOCK_SHAPES:
with self.subTest(shape=(m, n, k)):
layer, _ = self._build_layer(n, k)
layer.quant_method.process_weights_after_loading(layer)
x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16)
q, s = fp8_utils.sglang_per_token_group_quant_fp8(
x,
128,
column_major_scales=True,
scale_tma_aligned=True,
scale_ue8m0=True,
)
expected, _ = layer(x)
actual, _ = layer((q, s))
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
linear = fp8_utils.deepgemm_w8a8_block_fp8_linear_with_fallback
for bad_scale in (s.new_empty(s.shape), s[:, :-1], s.cpu()):
with self.assertRaisesRegex(ValueError, "packed UE8M0"):
linear(
q,
layer.weight,
[128, 128],
layer.weight_scale_inv,
bad_scale,
)
# This shape normally takes Triton, which cannot read packed A scales.
with self.assertRaisesRegex(ValueError, "packed UE8M0"):
linear(
q, layer.weight[:65], [128, 128], layer.weight_scale_inv, s
)
@unittest.skipUnless(
get_device_sm() == 90, "SM90 UE8M0 activation-scale regression"
)