From bb9e9d5632e33fb10c7e6e4a026df749fca7017a Mon Sep 17 00:00:00 2001 From: Jayden Yu <48167665+JiaFengYu@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:49:39 -0700 Subject: [PATCH] [Perf] mlp: Fuse dense SwiGLU with block-FP8 quantization on decode (#45219) Co-authored-by: Jayden Yu Co-authored-by: Xiaoyu Zhang <1182563586@qq.com> --- .../jit/csrc/gemm/per_token_group_quant.cuh | 25 +- .../ops/quantization/per_token_group_quant.py | 33 +- .../srt/layers/quantization/fp8_activation.py | 119 +++++++ .../srt/layers/quantization/fp8_utils.py | 61 +++- python/sglang/srt/models/llama.py | 11 +- python/sglang/srt/models/qwen2.py | 11 +- .../kernels/bench_dense_swiglu_quant.py | 147 +++++++++ .../bench_per_token_group_quant.py | 21 +- .../test_per_token_group_quant.py | 127 ++++++- .../quantization/test_dense_swiglu_fp8.py | 312 ++++++++++++++++++ .../test_fp8_blockwise_linear_backends.py | 40 +++ 11 files changed, 878 insertions(+), 29 deletions(-) create mode 100644 python/sglang/srt/layers/quantization/fp8_activation.py create mode 100644 test/manual/kernels/bench_dense_swiglu_quant.py create mode 100644 test/registered/unit/layers/quantization/test_dense_swiglu_fp8.py diff --git a/python/sglang/kernels/jit/csrc/gemm/per_token_group_quant.cuh b/python/sglang/kernels/jit/csrc/gemm/per_token_group_quant.cuh index 002dad73b5..6aecf5105a 100644 --- a/python/sglang/kernels/jit/csrc/gemm/per_token_group_quant.cuh +++ b/python/sglang/kernels/jit/csrc/gemm/per_token_group_quant.cuh @@ -238,7 +238,8 @@ template < bool kUe8m0_, bool kRowMajor_, bool kAligned_, - bool kFuseSiluAndMul_> + bool kFuseSiluAndMul_, + bool kDenseSiluAndMul = false> struct QuantTrait { // rename using InputType = InputType_; @@ -253,6 +254,7 @@ struct QuantTrait { static constexpr uint32_t kNumLanes = kGroupSize / kVecSize; static_assert(sizeof(InputType) == 2 || sizeof(InputType) == 4, "inputs must be 16-bit (bf16/fp16) or fp32"); static_assert(sizeof(InputType) == 2 || !kFuseSiluAndMul, "fp32 inputs do not implement the fused silu"); + static_assert(!kDenseSiluAndMul || (kFuseSiluAndMul && kGroupSize == 128 && std::is_same_v)); static_assert(16 <= kGroupSize && kGroupSize <= 256, "supported group sizes are 16..256"); static_assert(kGroupSize % kVecSize == 0 && 1 <= kNumLanes && kNumLanes <= device::kWarpThreads); static_assert(!kUe8m0 || std::is_same_v, "ue8m0 scales imply fp8 output"); @@ -302,8 +304,15 @@ struct QuantTrait { #pragma unroll for (uint32_t i = 0; i < kVecSize / 2; ++i) { const auto gate = cast(in[i]); - const auto act = cast(float2{detail::silu(gate.x), detail::silu(gate.y)}); - in[i] = __hmul2(act, up[i]); + if constexpr (kDenseSiluAndMul) { + // Match dense activation.cuh: round the product, not SiLU, to T. + const auto up_f32 = cast(up[i]); + in[i] = cast( + float2{(gate.x / (1.0f + expf(-gate.x))) * up_f32.x, (gate.y / (1.0f + expf(-gate.y))) * up_f32.y}); + } else { + const auto act = cast(float2{detail::silu(gate.x), detail::silu(gate.y)}); + in[i] = __hmul2(act, up[i]); + } } } @@ -342,7 +351,9 @@ struct QuantTrait { } else { // fp32 scale: multiply in fp32 (hmul2 brings too much precision loss) scale_inv = raw_scale; - const float quant_scale = kMaxValue / amax; + // Dense mode uses activation's compiler flags, but quantization must + // retain the fast division used by the standalone quantizer. + const float quant_scale = kDenseSiluAndMul ? __fdividef(kMaxValue, amax) : kMaxValue / amax; const float2 quant_scale2 = {quant_scale, quant_scale}; #pragma unroll for (uint32_t i = 0; i < kVecSize / 2; ++i) { @@ -621,9 +632,11 @@ template < bool kRowMajor, bool kAligned, bool kFuseSiluAndMul, - bool kUsePDL> + bool kUsePDL, + bool kDenseSiluAndMul = false> struct PerTokenGroupQuantFlatKernel { - using Trait = QuantTrait; + using Trait = + QuantTrait; static void run(tvm::ffi::TensorView input, tvm::ffi::TensorView output_q, tvm::ffi::TensorView output_s) { const auto ctx = build_quant_context(input, output_q, output_s); diff --git a/python/sglang/kernels/ops/quantization/per_token_group_quant.py b/python/sglang/kernels/ops/quantization/per_token_group_quant.py index fcb2133e2d..17832f332d 100644 --- a/python/sglang/kernels/ops/quantization/per_token_group_quant.py +++ b/python/sglang/kernels/ops/quantization/per_token_group_quant.py @@ -1,11 +1,12 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING, Literal, Optional, Tuple import torch from sglang.kernels.jit.utils import ( cache_once, + get_activation_cuda_cflags, is_arch_support_pdl, load_jit, make_cpp_args, @@ -33,10 +34,26 @@ def _jit_module( masked_layout: bool, use_pdl: bool, sorted_layout: bool = False, + silu_and_mul_mode: str = "legacy", ) -> Module: assert in_dtype in _SUPPORTED_INPUT_DTYPES assert out_dtype in _SUPPORTED_OUTPUT_DTYPES assert group_size in _SUPPORTED_GROUP_SIZES + if silu_and_mul_mode not in ("legacy", "dense"): + raise ValueError(f"Unknown silu_and_mul_mode: {silu_and_mul_mode}") + dense = silu_and_mul_mode == "dense" + if dense and not ( + fuse_silu_and_mul + and in_dtype in (torch.bfloat16, torch.float16) + and out_dtype == torch.float8_e4m3fn + and group_size == 128 + and not masked_layout + and not sorted_layout + ): + raise ValueError( + "dense mode requires flat BF16/FP16 SiLU+mul, FP8 output and group_size=128" + ) + # Only the flat dense variant adds a flag; legacy launcher arguments stay intact. trait_args = make_cpp_args( in_dtype, out_dtype, @@ -46,6 +63,7 @@ def _jit_module( aligned, fuse_silu_and_mul, use_pdl, + *([True] if dense else []), ) assert not (masked_layout and sorted_layout) layout = "masked" if masked_layout else ("sorted" if sorted_layout else "flat") @@ -56,7 +74,9 @@ def _jit_module( layout, cuda_files=["gemm/per_token_group_quant.cuh"], cuda_wrappers=[("per_token_group_quant", f"{launcher}<{trait_args}>::run")], - extra_cuda_cflags=["--use_fast_math"], + extra_cuda_cflags=get_activation_cuda_cflags() + if dense + else ["--use_fast_math"], ) @@ -97,6 +117,7 @@ def _per_token_group_quant_custom_op( sorted_token_ids: Optional[torch.Tensor] = None, num_tokens_post_padded: Optional[torch.Tensor] = None, num_valid_tokens: int = 0, + silu_and_mul_mode: str = "legacy", ) -> None: num_groups = output_q.shape[-1] // group_size row_major, aligned = _infer_scale_layout(output_s, scale_ue8m0, num_groups) @@ -111,6 +132,7 @@ def _per_token_group_quant_custom_op( masked_m is not None, is_arch_support_pdl(), sorted_token_ids is not None, + silu_and_mul_mode, ) if masked_m is not None: module.per_token_group_quant( @@ -186,6 +208,7 @@ def per_token_group_quant( sorted_token_ids: Optional[torch.Tensor] = None, num_tokens_post_padded: Optional[torch.Tensor] = None, num_valid_tokens: int = 0, + silu_and_mul_mode: Literal["legacy", "dense"] = "legacy", ) -> Tuple[torch.Tensor, torch.Tensor]: """Per-token-group quantization. Returns ``(output_q, output_s)``. @@ -207,6 +230,11 @@ def per_token_group_quant( int32 contiguous -> row-major UE8M0 bytes packed 4-per-int32 The packed layouts require ``scale_ue8m0=True``. + With ``fuse_silu_and_mul=True``, ``silu_and_mul_mode="dense"`` matches the + dense CUDA activation: FP32 SiLU and multiply, then one source-dtype round + before quantizing. The default retains the existing early SiLU rounding. + Dense mode supports flat BF16/FP16 inputs, FP8 output and groups of 128. + ``expected_m`` (masked only) is an optional expected-tokens-per-expert hint. ``sorted_token_ids`` selects the sorted layout: ``input`` rows follow @@ -242,5 +270,6 @@ def per_token_group_quant( sorted_token_ids=sorted_token_ids, num_tokens_post_padded=num_tokens_post_padded, num_valid_tokens=num_valid_tokens, + silu_and_mul_mode=silu_and_mul_mode, ) return output_q, output_s diff --git a/python/sglang/srt/layers/quantization/fp8_activation.py b/python/sglang/srt/layers/quantization/fp8_activation.py new file mode 100644 index 0000000000..ff839f992e --- /dev/null +++ b/python/sglang/srt/layers/quantization/fp8_activation.py @@ -0,0 +1,119 @@ +"""Dense activation/quantization fusion for the block-FP8 DeepGEMM path.""" + +from __future__ import annotations + +from functools import partial +from typing import TYPE_CHECKING + +import torch + +from sglang.kernels.fused_op import get_fused_op_backend +from sglang.srt.runtime_context import ( + get_exec, + get_forward, + get_lora, + get_parallel, + get_platform, +) +from sglang.srt.utils import is_cuda + +if TYPE_CHECKING: + from sglang.srt.layers.activation import SiluAndMul + from sglang.srt.layers.linear import RowParallelLinear + + +class DenseSiluAndMulQuant: + """Keep the normal activation unless down_proj can consume our FP8 tuple.""" + + def __init__(self, down_proj: RowParallelLinear) -> None: + self.enabled = False + self.down_proj = down_proj + self.quant_method = down_proj.quant_method + if not (is_cuda() and get_platform().is_sm100 and get_parallel().tp_size == 1): + return + + from sglang.srt.layers import deep_gemm_wrapper + from sglang.srt.layers.linear import RowParallelLinear + from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod + from sglang.srt.layers.quantization.fp8_utils import ( + _deepgemm_block_fp8_supported, + deepgemm_w8a8_block_fp8_linear_with_fallback, + ) + + method = self.quant_method + if not ( + type(down_proj) is RowParallelLinear + and type(method) is Fp8LinearMethod + and method.weight_block_size == [128, 128] + and method.quant_config.activation_scheme == "dynamic" + and not (method.use_marlin or method.use_mxfp8 or method.block_fp8_as_mxfp8) + and deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0 + and down_proj.input_is_parallel + ): + return + backend = method.w8a8_block_fp8_linear + if isinstance(backend, partial): + if backend.args or set(backend.keywords) - {"act_scale_ue8m0"}: + return + backend = backend.func + # The supported loader preserves matrix geometry, not tensor identity/device. + self.weight_shape = (down_proj.output_size, down_proj.input_size_per_partition) + self.enabled = ( + backend is deepgemm_w8a8_block_fp8_linear_with_fallback + and _deepgemm_block_fp8_supported( + self.weight_shape, method.weight_block_size, torch.bfloat16 + ) + ) + + def forward_decode( + self, + x: torch.Tensor, + act_fn: SiluAndMul, + down_proj: RowParallelLinear, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + """Try fusion for decode; callers must bypass this helper in other modes.""" + if not self.enabled or torch.compiler.is_compiling(): + return act_fn(x) + + from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled + from sglang.srt.layers.activation import SiluAndMul + + deterministic = get_exec().deterministic + if ( + down_proj is not self.down_proj + or down_proj.quant_method is not self.quant_method + or down_proj.weight.shape != self.weight_shape + or type(act_fn) is not SiluAndMul + or (act_fn._forward_method or act_fn._resolve_forward_method()) + != act_fn.forward_cuda + or act_fn.is_torch_compile + or get_fused_op_backend() is not None + or deterministic.enable_deterministic_inference + or deterministic.rl_on_policy_target is not None + or is_batch_invariant_mode_enabled() + or get_lora().enable_lora + or get_forward().sp_active + or x.dtype != torch.bfloat16 + or not x.is_cuda + or x.ndim != 2 + or x.shape[0] == 0 + or not x.is_contiguous() + or x.data_ptr() % 32 != 0 + or x.stride(0) % 16 != 0 + or x.device != down_proj.weight.device + or x.shape[1] != 2 * self.weight_shape[1] + ): + return act_fn(x) + + from sglang.kernels.ops.quantization.per_token_group_quant import ( + per_token_group_quant, + ) + + return per_token_group_quant( + x, + group_size=128, + scale_ue8m0=True, + column_major_scales=True, + fuse_silu_and_mul=True, + silu_and_mul_mode="dense", + ) diff --git a/python/sglang/srt/layers/quantization/fp8_utils.py b/python/sglang/srt/layers/quantization/fp8_utils.py index 11b36986fe..cf4368708d 100755 --- a/python/sglang/srt/layers/quantization/fp8_utils.py +++ b/python/sglang/srt/layers/quantization/fp8_utils.py @@ -1453,6 +1453,25 @@ def _deepgemm_block_fp8_supported( return list(block_size) == [128, 128] and n % 64 == 0 and k % 128 == 0 +def _deepgemm_packed_input_valid(input, weight, input_scale, weight_scale) -> bool: + """Validate the group-128, column-major TMA layout for packed activations.""" + if input.ndim != 2 or weight.ndim != 2: + return False + m, k = input.shape + packed_groups = ceil_div(k // 128, 4) + padded_m = ceil_align(m, 4) + return ( + input.is_cuda + and input.is_contiguous() + and k == weight.shape[1] + and input.device == weight.device == input_scale.device == weight_scale.device + and input_scale.shape == (m, packed_groups) + and input_scale.stride() == (1, padded_m) + and input.data_ptr() % 16 == 0 + and input_scale.data_ptr() % 16 == 0 + ) + + def deepgemm_w8a8_block_fp8_linear_with_fallback( input: torch.Tensor, weight: torch.Tensor, @@ -1463,18 +1482,36 @@ def deepgemm_w8a8_block_fp8_linear_with_fallback( act_scale_ue8m0: bool = False, weight_bf16: Optional[torch.Tensor] = None, ) -> torch.Tensor: - if input_scale is not None: - # Block32 pre-quantized input reaches Triton below on UE8M0 DeepGEMM. - assert ( - list(block_size) == [32, 32] or not deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0 - ) - assert input.dtype == torch.float8_e4m3fn output_dtype = torch.bfloat16 if input_scale is not None else input.dtype - - # TODO: https://github.com/sgl-project/sglang/pull/6890#issuecomment-2943395737 - if not _deepgemm_block_fp8_supported( + deepgemm_supported = weight.ndim == 2 and _deepgemm_block_fp8_supported( weight.shape, block_size, output_dtype - ) or _ue8m0_deepgemm_skips_block32(block_size, weight_scale, input_scale): + ) + if input_scale is not None: + if input_scale.dtype == torch.int32: + # Packed activation scales cannot go through the Triton fallback. + if not ( + deepgemm_supported + and deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0 + and list(block_size) == [128, 128] + and _deepgemm_packed_input_valid( + input, weight, input_scale, weight_scale + ) + ): + raise ValueError( + "Pre-quantized DeepGEMM block-FP8 input requires supported " + "[128, 128] weights and TMA-aligned, column-major packed UE8M0 scales" + ) + else: + # Block32 pre-quantized input uses the Triton fallback. + assert ( + list(block_size) == [32, 32] + or not deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0 + ) + assert input.dtype == torch.float8_e4m3fn + # TODO: https://github.com/sgl-project/sglang/pull/6890#issuecomment-2943395737 + if not deepgemm_supported or _ue8m0_deepgemm_skips_block32( + block_size, weight_scale, input_scale + ): # fall back to triton # If weight_scale is in UE8M0 packed format (int32), convert back to float32 # UE8M0 format has shape (N, K//block_k//4) with dtype int32 @@ -1495,8 +1532,8 @@ def deepgemm_w8a8_block_fp8_linear_with_fallback( ) if input_scale is not None: - # Pre-quantized activations retain their per-token group size and - # fp32 scales. DeepGEMM aligns scale storage for TMA when needed. + # Pre-quantized activations retain their group size and scale format. + # DeepGEMM aligns fp32 scale storage for TMA when needed. input_2d = input.view(-1, input.shape[-1]) output = w8a8_block_fp8_matmul_deepgemm( input_2d, diff --git a/python/sglang/srt/models/llama.py b/python/sglang/srt/models/llama.py index 07cf4a1ae5..7b7a045df1 100644 --- a/python/sglang/srt/models/llama.py +++ b/python/sglang/srt/models/llama.py @@ -49,6 +49,7 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput from sglang.srt.layers.pooler import Pooler, PoolingType from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.layers.quantization.fp8_activation import DenseSiluAndMulQuant from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.layers.rotary_embedding import get_rope from sglang.srt.layers.utils import PPMissingLayer, get_layer_id @@ -118,6 +119,7 @@ class LlamaMLP(nn.Module): ) self.act_fn = SiluAndMul() self.dual_gemm = DualGemm(self.gate_up_proj, self.down_proj, hidden_size) + self.activation_quant = DenseSiluAndMulQuant(self.down_proj) def forward( self, @@ -128,7 +130,12 @@ class LlamaMLP(nn.Module): x = self.dual_gemm(x, self.gate_up_proj) else: gate_up, _ = self.gate_up_proj(x) - x = self.act_fn(gate_up) + if forward_batch is not None and forward_batch.forward_mode.is_decode(): + x = self.activation_quant.forward_decode( + gate_up, self.act_fn, self.down_proj + ) + else: + x = self.act_fn(gate_up) x, _ = self.down_proj(x) return x @@ -404,7 +411,7 @@ class LlamaDecoderLayer(nn.Module): # Fully Connected hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - hidden_states = self.mlp(hidden_states) + hidden_states = self.mlp(hidden_states, forward_batch=forward_batch) return self.ffn_boundary.finish(hidden_states, forward_batch) diff --git a/python/sglang/srt/models/qwen2.py b/python/sglang/srt/models/qwen2.py index 9f80581db0..1150397f44 100644 --- a/python/sglang/srt/models/qwen2.py +++ b/python/sglang/srt/models/qwen2.py @@ -46,6 +46,7 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.pooler import Pooler, PoolingType from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.layers.quantization.fp8_activation import DenseSiluAndMulQuant from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.layers.rotary_embedding import get_rope from sglang.srt.layers.utils import PPMissingLayer, get_layer_id @@ -102,6 +103,7 @@ class Qwen2MLP(nn.Module): ) self.act_fn = SiluAndMul() self.dual_gemm = DualGemm(self.gate_up_proj, self.down_proj, hidden_size) + self.activation_quant = DenseSiluAndMulQuant(self.down_proj) def forward( self, @@ -115,7 +117,12 @@ class Qwen2MLP(nn.Module): x = self.dual_gemm(x, self.gate_up_proj) else: gate_up, _ = self.gate_up_proj(x) - x = self.act_fn(gate_up) + if forward_batch is not None and forward_batch.forward_mode.is_decode(): + x = self.activation_quant.forward_decode( + gate_up, self.act_fn, self.down_proj + ) + else: + x = self.act_fn(gate_up) x, _ = self.down_proj(x, forward_batch=forward_batch) return x @@ -339,7 +346,7 @@ class Qwen2DecoderLayer(nn.Module): # Fully Connected hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - hidden_states = self.mlp(hidden_states) + hidden_states = self.mlp(hidden_states, forward_batch=forward_batch) return self.ffn_boundary.finish(hidden_states, forward_batch) diff --git a/test/manual/kernels/bench_dense_swiglu_quant.py b/test/manual/kernels/bench_dense_swiglu_quant.py new file mode 100644 index 0000000000..663e1086ac --- /dev/null +++ b/test/manual/kernels/bench_dense_swiglu_quant.py @@ -0,0 +1,147 @@ +"""Paired B200 MLP timing for dense SwiGLU fusion. + +Synthetic weights test the operator pipeline, not model quality or serving TBT. +Operator parity and the opt-in numerical sweep live in test_per_token_group_quant.py. +""" + +import argparse +import json +import random +import statistics +import subprocess + +import torch + +from sglang.kernels.jit.benchmark import marker +from sglang.kernels.ops.activation.activation import silu_and_mul +from sglang.kernels.ops.quantization.per_token_group_quant import per_token_group_quant + + +def mlp(x, gate_up, gate_scale, down, down_scale, *, fused): + from sglang.srt.layers.quantization.fp8_utils import ( + deepgemm_w8a8_block_fp8_linear_with_fallback as linear, + ) + + projected = linear(x, gate_up, [128, 128], gate_scale) + if fused: + q, s = per_token_group_quant( + projected, + scale_ue8m0=True, + column_major_scales=True, + fuse_silu_and_mul=True, + silu_and_mul_mode="dense", + ) + return linear(q, down, [128, 128], down_scale, s) + return linear(silu_and_mul(projected), down, [128, 128], down_scale) + + +def benchmark(args): + from functools import partial + + torch.manual_seed(20261017) + order = random.Random(17) + for m in args.batches: + k, h = args.intermediate_size, 4096 + assert k % 512 == 0 + x = torch.randn(m, h, device="cuda", dtype=torch.bfloat16) + up = (torch.randn(2 * k, h, device="cuda") * 0.01).to(torch.float8_e4m3fn) + down = (torch.randn(h, k, device="cuda") * 0.01).to(torch.float8_e4m3fn) + # Packed exponent 127 represents scale=1, using DeepGEMM's weight layout. + us = torch.full( + (h // 512, 2 * k), 0x7F7F7F7F, device="cuda", dtype=torch.int32 + ).T + ds = torch.full((k // 512, h), 0x7F7F7F7F, device="cuda", dtype=torch.int32).T + inputs = (x, up, us, down, ds) + variants = { + "eager": partial(mlp, fused=False), + "fused": partial(mlp, fused=True), + "control": partial(mlp, fused=False), + } + torch.testing.assert_close( + variants["eager"](*inputs), variants["fused"](*inputs), rtol=0, atol=0 + ) + if args.profile: + for name in ("eager", "fused"): + with torch.profiler.profile( + activities=[ + torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ] + ) as prof: + variants[name](*inputs) + torch.cuda.synchronize() + kernels = [ + e.name + for e in prof.events() + if e.device_type == torch.autograd.DeviceType.CUDA + ] + print( + json.dumps({"batch": m, "variant": name, "CUDA_events": kernels}), + flush=True, + ) + samples = {name: [] for name in variants} + for _ in range(args.rounds): + names = list(variants) + order.shuffle(names) + for name in names: + # Rotate all inputs, including both GEMM weights, beyond L2. + result = marker.do_bench( + variants[name], + input_args=inputs, + metrics=(0.5,), + estimated_time_ms=30, + warmup_iters=3, + graph_clone_args="all", + disable_log_bandwidth=True, + ) + samples[name].append(result.times[0] * 1e6) + eager, fused, control = (samples[n] for n in ("eager", "fused", "control")) + print( + json.dumps( + { + "batch": m, + "intermediate_size": k, + "rounds": args.rounds, + "cold_L2": True, + "raw_us": samples, + "median_us": {n: statistics.median(s) for n, s in samples.items()}, + "paired_wins": sum(a > b for a, b in zip(eager, fused)), + "paired_saving_us": statistics.median( + a - b for a, b in zip(eager, fused) + ), + "control_adjusted_saving_us": statistics.median( + (a + c) / 2 - b for a, b, c in zip(eager, fused, control) + ), + "AA_median_abs_difference_us": statistics.median( + abs(a - c) for a, c in zip(eager, control) + ), + } + ), + flush=True, + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--rounds", type=int, default=10) + parser.add_argument("--batches", type=int, nargs="+", default=[1, 32, 128]) + parser.add_argument("--intermediate-size", type=int, default=14336) + parser.add_argument("--profile", action="store_true") + args = parser.parse_args() + print( + json.dumps( + { + "base_commit": subprocess.check_output( + ["git", "rev-parse", "HEAD"], text=True + ).strip(), + "worktree_dirty": bool( + subprocess.check_output(["git", "status", "--porcelain"], text=True) + ), + "torch": torch.__version__, + "gpu": torch.cuda.get_device_name(), + } + ), + flush=True, + ) + with torch.inference_mode(): + benchmark(args) diff --git a/test/registered/kernels/benchmark/quantization/bench_per_token_group_quant.py b/test/registered/kernels/benchmark/quantization/bench_per_token_group_quant.py index 8451976c82..b7eb48531a 100644 --- a/test/registered/kernels/benchmark/quantization/bench_per_token_group_quant.py +++ b/test/registered/kernels/benchmark/quantization/bench_per_token_group_quant.py @@ -1,5 +1,6 @@ from sglang.kernels.jit.benchmark import marker from sglang.kernels.jit.benchmark.utils import create_empty, create_random +from sglang.kernels.ops.activation.activation import silu_and_mul from sglang.kernels.ops.quantization.fp8_kernel import ( create_per_token_group_quant_fp8_output_scale, fp8_dtype, @@ -18,7 +19,7 @@ from sglang.kernels.ops.quantization.per_token_group_quant_8bit_v2 import ( from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci( - est_time=25, stage="base-b-kernel-benchmark", runner_config="1-gpu-large" + est_time=35, stage="base-b-kernel-benchmark", runner_config="1-gpu-large" ) HIDDEN = 2048 @@ -73,5 +74,23 @@ def benchmark(group_size: int, layout: str, num_tokens: int, impl: str): ) +def _dense_activation_quant(x, fused): + return per_token_group_quant( + x if fused else silu_and_mul(x), + fuse_silu_and_mul=fused, + silu_and_mul_mode="dense" if fused else "legacy", + scale_ue8m0=True, + column_major_scales=True, + ) + + +@marker.parametrize("num_tokens", [1, 32, 128], ci_vals=[1, 32]) +@marker.benchmark("fused", [False, True]) +def benchmark_dense(num_tokens: int, fused: bool): + x = create_random(num_tokens, 8192) + return marker.do_bench(_dense_activation_quant, input_args=(x, fused)) + + if __name__ == "__main__": benchmark.run() + benchmark_dense.run() diff --git a/test/registered/kernels/ops/quantization/test_per_token_group_quant.py b/test/registered/kernels/ops/quantization/test_per_token_group_quant.py index a5b717f573..d9c9278dee 100644 --- a/test/registered/kernels/ops/quantization/test_per_token_group_quant.py +++ b/test/registered/kernels/ops/quantization/test_per_token_group_quant.py @@ -21,7 +21,8 @@ import itertools import pytest import torch -from sglang.kernels.jit.utils import get_ci_test_range +from sglang.kernels.jit.utils import get_ci_test_range, should_run_full_tests +from sglang.kernels.ops.activation.activation import silu_and_mul from sglang.kernels.ops.quantization.fp8_kernel import ( create_per_token_group_quant_fp8_output_scale, fp8_dtype, @@ -32,9 +33,9 @@ from sglang.kernels.ops.quantization.per_token_group_quant import ( ) from sglang.test.ci.ci_register import register_cuda_ci -register_cuda_ci(est_time=65, stage="base-b-kernel-unit", runner_config="1-gpu-large") -register_cuda_ci(est_time=90, stage="base-b-kernel-unit", runner_config="4-gpu-b200") -register_cuda_ci(est_time=120, stage="nightly", runner_config="1-gpu-large") +register_cuda_ci(est_time=120, stage="base-b-kernel-unit", runner_config="1-gpu-large") +register_cuda_ci(est_time=200, stage="base-b-kernel-unit", runner_config="4-gpu-b200") +register_cuda_ci(est_time=180, stage="nightly", runner_config="1-gpu-large") G = 128 FMAX = float(fp8_max) # 448 for e4m3 @@ -527,6 +528,124 @@ def test_fused_silu(scale_ue8m0, column_major): assert _dequant_rel_err(x_q, deq_scale, act, G) < 0.05 +def _check_dense_silu(x, scale_ue8m0, column_major): + original = x.view(torch.int16).clone() + kwargs = dict(scale_ue8m0=scale_ue8m0, column_major_scales=column_major) + expected_q, expected_s = per_token_group_quant(silu_and_mul(x), **kwargs) + q, s = per_token_group_quant( + x, fuse_silu_and_mul=True, silu_and_mul_mode="dense", **kwargs + ) + assert torch.equal(q.view(torch.uint8), expected_q.view(torch.uint8)) + assert torch.equal(s.view(torch.int32), expected_s.view(torch.int32)) + assert torch.equal(x.view(torch.int16), original) + + +@pytest.mark.parametrize( + "dtype,scale_ue8m0,column_major", + get_ci_test_range( + list( + itertools.product( + [torch.bfloat16, torch.float16], [False, True], [False, True] + ) + ), + [ + (torch.bfloat16, True, True), + (torch.float16, True, True), + (torch.bfloat16, False, False), + ], + ), +) +@pytest.mark.parametrize( + "shape", + get_ci_test_range( + [(1, 768), (3, 4096), (32, 12288), (32, 14336), (128, 28672)], + [(1, 768), (32, 14336)], + ), +) +def test_dense_silu_bitexact(dtype, shape, scale_ue8m0, column_major): + """Dense fusion preserves the separately stored activation's rounding. + + The legacy fused mode rounds SiLU before multiplication and fails this + comparison. Use the real CUDA activation as the compiler-math reference. + """ + torch.manual_seed(42) + m, h = shape + x = torch.randn(m, 2 * h, device="cuda", dtype=dtype) * 3 + x[0, :8] = torch.tensor( + [0.0, -0.0, 1e-6, -1e-6, 100, -100, float("inf"), float("nan")], + device="cuda", + dtype=dtype, + ) + _check_dense_silu(x, scale_ue8m0, column_major) + + +@pytest.mark.skipif( + not should_run_full_tests(), + reason="Set SGLANG_JIT_KERNEL_RUN_FULL_TESTS=1 for the sweep", +) +def test_dense_silu_extended(): + """Opt-in amplitude/layout sweep and all 65,536 BF16/FP16 gate encodings.""" + torch.manual_seed(20261017) + layouts = list(itertools.product((False, True), repeat=2)) + for dtype in (torch.bfloat16, torch.float16): + for (m, k), amplitude in itertools.product( + ((1, 768), (3, 4096), (32, 12288), (32, 14336), (128, 28672)), + (0, 0.001, 1, 10, 100), + ): + x = (torch.randn(m, 2 * k, device="cuda") * amplitude).to(dtype) + for ue8m0, column in layouts: + _check_dense_silu(x, ue8m0, column) + # Signed zeros, subnormals, finite extremes, infinities and NaNs. + gate = ( + torch.arange(65536, device="cuda", dtype=torch.int32) + .to(torch.int16) + .view(dtype) + .reshape(64, 1024) + ) + for up_value in (0, 0.25, 1, 1.125, -2.25): + x = torch.cat((gate, torch.full_like(gate, up_value)), dim=1) + for ue8m0, column in layouts: + _check_dense_silu(x, ue8m0, column) + + +def test_dense_silu_graph_replay(): + """Replays must use new input values and retain the allocated output buffers.""" + x = torch.randn(3, 1536, device="cuda", dtype=torch.bfloat16) + kwargs = dict(scale_ue8m0=True, column_major_scales=True) + q, s = per_token_group_quant( + x, fuse_silu_and_mul=True, silu_and_mul_mode="dense", **kwargs + ) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + per_token_group_quant( + x, q, s, fuse_silu_and_mul=True, silu_and_mul_mode="dense", **kwargs + ) + for amplitude in (0, 0.001, 10): + x.normal_().mul_(amplitude) + graph.replay() + expected_q, expected_s = per_token_group_quant(silu_and_mul(x), **kwargs) + assert torch.equal(q.view(torch.uint8), expected_q.view(torch.uint8)) + assert torch.equal(s, expected_s) + + +@pytest.mark.parametrize( + "overrides", + [ + {"silu_and_mul_mode": "unknown"}, + {"fuse_silu_and_mul": False}, + {"out_dtype": torch.int8}, + {"group_size": 64}, + {"masked_m": torch.tensor([1], dtype=torch.int32)}, + ], +) +def test_dense_silu_invalid_mode(overrides): + x = torch.zeros(1, 256, device="cuda", dtype=torch.bfloat16) + kwargs = dict(fuse_silu_and_mul=True, silu_and_mul_mode="dense") + kwargs.update(overrides) + with pytest.raises(ValueError, match="mode"): + per_token_group_quant(x, **kwargs) + + # --------------------------------------------------------------------------- # # Masked EP-MoE schedule. # --------------------------------------------------------------------------- # diff --git a/test/registered/unit/layers/quantization/test_dense_swiglu_fp8.py b/test/registered/unit/layers/quantization/test_dense_swiglu_fp8.py new file mode 100644 index 0000000000..17c0b501bc --- /dev/null +++ b/test/registered/unit/layers/quantization/test_dense_swiglu_fp8.py @@ -0,0 +1,312 @@ +"""Dense SwiGLU/block-FP8 integration: real MLPs, dispatch and graph replay.""" + +import unittest +from types import SimpleNamespace +from unittest import mock + +import torch + +from sglang.srt.layers.quantization import fp8_utils +from sglang.srt.layers.quantization.fp8 import Fp8Config +from sglang.srt.layers.quantization.fp8_utils import Fp8GemmRunnerBackend +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.model_loader.utils import set_default_torch_dtype +from sglang.srt.utils import get_device_sm +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.layer_ut_utils import init_single_process_dist +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=45, stage="base-b", runner_config="4-gpu-b200") + + +@unittest.skipUnless(get_device_sm() in (100, 103), "Dense block-FP8 fusion on SM100") +@mock.patch.object(fp8_utils, "FP8_GEMM_RUNNER_BACKEND", Fp8GemmRunnerBackend.DEEP_GEMM) +class TestDenseSiluAndMulQuant(CustomTestCase): + @classmethod + def setUpClass(cls): + init_single_process_dist() + + def _make_mlp(self, model): + from sglang.srt.models.llama import LlamaMLP + from sglang.srt.models.qwen2 import Qwen2MLP + + config = Fp8Config( + is_checkpoint_fp8_serialized=True, + activation_scheme="dynamic", + weight_block_size=[128, 128], + ) + with set_default_torch_dtype(torch.bfloat16): + mlp = {"llama": LlamaMLP, "qwen": Qwen2MLP}[model]( + 512, + 768, + "silu", + quant_config=config, + reduce_results=False, + ).cuda() + with torch.no_grad(): + for layer in (mlp.gate_up_proj, mlp.down_proj): + weight = ( + torch.randn(layer.weight.shape, device="cuda", dtype=torch.bfloat16) + * 0.02 + ) + layer.weight.copy_(weight.to(torch.float8_e4m3fn)) + layer.weight_scale_inv.fill_(1) + layer.quant_method.process_weights_after_loading(layer) + return mlp + + def test_mlp_parity_and_graph_replay(self): + """Cold first-decode dispatch and graph replay must match the unfused MLP.""" + batch = SimpleNamespace(forward_mode=ForwardMode.DECODE) + torch.manual_seed(123) + with torch.inference_mode(): + for model in ("llama", "qwen"): + mlp = self._make_mlp(model) + self.assertTrue(mlp.activation_quant.enabled) + self.assertIsNone(mlp.dual_gemm.mode) + self.assertIsNone(mlp.act_fn._forward_method) + for m in (1, 3, 32, 128): + with self.subTest(model=model, m=m): + x = torch.randn(m, 512, device="cuda", dtype=torch.bfloat16) + gate_up, _ = mlp.gate_up_proj(x) + quantized = mlp.activation_quant.forward_decode( + gate_up, mlp.act_fn, mlp.down_proj + ) + self.assertIsInstance(quantized, tuple) + expected = mlp(x) + actual = mlp(x, batch) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + actual = mlp(x, batch) + for amplitude in (0, 0.1, 3): + x.normal_().mul_(amplitude) + graph.replay() + torch.testing.assert_close(actual, mlp(x), rtol=0, atol=0) + + @torch.inference_mode() + def test_non_decode_bypass(self): + """Every non-decode mode must avoid the helper and preserve MLP output.""" + for model in ("llama", "qwen"): + mlp = self._make_mlp(model) + x = torch.randn(1, 512, device="cuda", dtype=torch.bfloat16) + expected = mlp(x) + with mock.patch.object( + type(mlp.activation_quant), + "forward_decode", + side_effect=AssertionError("non-decode entered fusion dispatch"), + ): + for mode in (None, *ForwardMode): + if mode == ForwardMode.DECODE: + continue + with self.subTest(model=model, mode=mode): + batch = ( + SimpleNamespace(forward_mode=mode) + if mode is not None + else None + ) + torch.testing.assert_close( + mlp(x, batch), expected, rtol=0, atol=0 + ) + + @torch.inference_mode() + def test_decoder_handoff(self): + """Dropping forward_batch between the decoder and real MLP must fail.""" + from sglang.srt.models.llama import LlamaDecoderLayer + from sglang.srt.models.qwen2 import Qwen2DecoderLayer + + batch = SimpleNamespace(forward_mode=ForwardMode.DECODE) + boundary = SimpleNamespace( + prepare=lambda x, batch, **kw: x, finish=lambda x, batch: x + ) + for model, decoder_cls in ( + ("llama", LlamaDecoderLayer), + ("qwen", Qwen2DecoderLayer), + ): + mlp = self._make_mlp(model) + x = torch.randn(1, 512, device="cuda", dtype=torch.bfloat16) + expected = mlp(x) + # Only attention/communication are stubbed; run the real decoder and MLP. + decoder = SimpleNamespace( + attn_boundary=boundary, + ffn_boundary=boundary, + self_attn=lambda **kw: kw["hidden_states"], + mlp=mlp, + ) + with ( + self.subTest(model=model), + mock.patch.object( + mlp.act_fn, + "forward", + side_effect=AssertionError("decode used separate activation"), + ), + ): + actual = decoder_cls.forward(decoder, None, x, batch) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + def test_decode_fallback_overrides_and_inputs(self): + """Overrides and unsupported inputs must retain the original activation.""" + from sglang.kernels.fused_op import get_fused_op_backend, set_fused_op_backend + from sglang.kernels.spec import KernelBackend + from sglang.srt.runtime_context import get_exec, get_forward, get_lora + + mlp = self._make_mlp("llama") + x = torch.randn(3, 1536, device="cuda", dtype=torch.bfloat16) + + def check(*, value=x, projection=mlp.down_proj): + actual = mlp.activation_quant.forward_decode(value, mlp.act_fn, projection) + self.assertIsInstance(actual, torch.Tensor) + torch.testing.assert_close(actual, mlp.act_fn(value), rtol=0, atol=0) + + with get_forward().scoped(sp_active=True): + check() + with get_lora().override(enable_lora=True): + check() + with get_exec().deterministic.override(enable_deterministic_inference=True): + check() + with mock.patch.object( + mlp.act_fn, "_forward_method", mlp.act_fn.forward_native + ): + check() + old_backend = get_fused_op_backend() + try: + set_fused_op_backend(KernelBackend.TORCH) + check() + finally: + set_fused_op_backend(old_backend) + for dtype in (torch.float16, torch.float32): + check(value=x.to(dtype)) + # Contiguous single-row views may still have an unaligned row stride. + check(value=x[:1].as_strided((1, 1536), (1, 1))) + + wrapped_projection = SimpleNamespace( + quant_method=mlp.down_proj.quant_method, + weight=mlp.down_proj.weight, + ) + check(projection=wrapped_projection) + actual = mlp.activation_quant.forward_decode( + x, torch.nn.Identity(), mlp.down_proj + ) + self.assertIs(actual, x) + + mlp.act_fn.enter_torch_compile(x.shape[0]) + try: + check() + compiled = torch.compile( + lambda value: mlp.activation_quant.forward_decode( + value, mlp.act_fn, mlp.down_proj + ), + backend="eager", + fullgraph=True, + ) + torch.testing.assert_close( + compiled(x), mlp.act_fn.forward_native(x), rtol=0, atol=0 + ) + finally: + mlp.act_fn.leave_torch_compile() + + def test_constructor_gates(self): + """Other quantization/backends must retain activation tensors for down_proj.""" + from sglang.srt.layers.activation import SiluAndMul + from sglang.srt.layers.linear import RowParallelLinear + from sglang.srt.layers.quantization.fp8_activation import DenseSiluAndMulQuant + from sglang.srt.runtime_context import override_platform + from sglang.test.parallel_groups import parallel_scope + + cases = [ + (None, "deep_gemm"), + ({}, "deep_gemm"), + ({"weight_block_size": [128, 128]}, "triton"), + ({"use_mxfp8": True}, "auto"), + ({"weight_block_size": [32, 32], "scale_fmt": "ue8m0"}, "auto"), + ] + act = SiluAndMul() + x = torch.randn(3, 1536, device="cuda", dtype=torch.bfloat16) + expected = act(x) + for options, backend in cases: + config = ( + Fp8Config(is_checkpoint_fp8_serialized=True, **options) + if options is not None + else None + ) + with ( + self.subTest(config=options, backend=backend), + mock.patch.object( + fp8_utils, "FP8_GEMM_RUNNER_BACKEND", Fp8GemmRunnerBackend(backend) + ), + ): + layer = RowParallelLinear( + 768, + 512, + bias=False, + params_dtype=torch.bfloat16, + quant_config=config, + ).cuda() + helper = DenseSiluAndMulQuant(layer) + torch.testing.assert_close( + helper.forward_decode(x, act, layer), expected, rtol=0, atol=0 + ) + mlp = self._make_mlp("llama") + for scope in ( + parallel_scope(tp_size=2, attn_tp_size=2, moe_tp_size=2), + override_platform(is_sm100=False), + ): + with scope: + helper = DenseSiluAndMulQuant(mlp.down_proj) + torch.testing.assert_close( + helper.forward_decode(x, act, mlp.down_proj), expected, rtol=0, atol=0 + ) + + @torch.inference_mode() + def test_fixed_geometry_and_live_projection_guards(self): + """Cache support rules, but reject changed geometry/methods and other layers.""" + from sglang.srt.layers.linear import RowParallelLinear + from sglang.srt.layers.quantization.fp8_activation import DenseSiluAndMulQuant + + mlp = self._make_mlp("llama") + layer = mlp.down_proj + helper = mlp.activation_quant + x = torch.randn(3, 1536, device="cuda", dtype=torch.bfloat16) + expected = mlp.act_fn(x) + # Same-shape weight reloads remain eligible without redoing support checks. + with ( + mock.patch.object( + layer, "weight", torch.nn.Parameter(layer.weight.clone(), False) + ), + mock.patch.object( + fp8_utils, "_deepgemm_block_fp8_supported", side_effect=AssertionError + ), + ): + self.assertIsInstance(helper.forward_decode(x, mlp.act_fn, layer), tuple) + for attr, value in ( + ("weight", torch.nn.Parameter(layer.weight.reshape(1024, 384), False)), + ("quant_method", object()), + ): + with self.subTest(attr=attr), mock.patch.object(layer, attr, value): + torch.testing.assert_close( + helper.forward_decode(x, mlp.act_fn, layer), + expected, + rtol=0, + atol=0, + ) + for n, k in ((512, 768), (513, 768), (512, 769)): + other = RowParallelLinear( + k, + n, + bias=False, + params_dtype=torch.bfloat16, + quant_config=layer.quant_method.quant_config, + ) + with self.subTest(n=n, k=k): + self.assertEqual( + DenseSiluAndMulQuant(other).enabled, (n, k) == (512, 768) + ) + torch.testing.assert_close( + helper.forward_decode(x, mlp.act_fn, other), + expected, + rtol=0, + atol=0, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/quantization/test_fp8_blockwise_linear_backends.py b/test/registered/unit/layers/quantization/test_fp8_blockwise_linear_backends.py index 885ab4d199..68d07de502 100644 --- a/test/registered/unit/layers/quantization/test_fp8_blockwise_linear_backends.py +++ b/test/registered/unit/layers/quantization/test_fp8_blockwise_linear_backends.py @@ -192,6 +192,46 @@ class TestFp8BlockwiseLinearBackends(_LinearBackendCheck): def test_deep_gemm(self): self._run("deep_gemm") + @unittest.skipUnless( + get_device_sm() in (100, 103), "Packed block-FP8 scales on SM100" + ) + def test_deep_gemm_prequantized(self): + """Packed activations must skip requantization and preserve the GEMM result.""" + torch.manual_seed(9) + with mock.patch.object( + fp8_utils, "FP8_GEMM_RUNNER_BACKEND", Fp8GemmRunnerBackend.DEEP_GEMM + ): + for m, n, k in FP8_BLOCK_SHAPES: + with self.subTest(shape=(m, n, k)): + layer, _ = self._build_layer(n, k) + layer.quant_method.process_weights_after_loading(layer) + x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) + q, s = fp8_utils.sglang_per_token_group_quant_fp8( + x, + 128, + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + ) + expected, _ = layer(x) + actual, _ = layer((q, s)) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + linear = fp8_utils.deepgemm_w8a8_block_fp8_linear_with_fallback + for bad_scale in (s.new_empty(s.shape), s[:, :-1], s.cpu()): + with self.assertRaisesRegex(ValueError, "packed UE8M0"): + linear( + q, + layer.weight, + [128, 128], + layer.weight_scale_inv, + bad_scale, + ) + # This shape normally takes Triton, which cannot read packed A scales. + with self.assertRaisesRegex(ValueError, "packed UE8M0"): + linear( + q, layer.weight[:65], [128, 128], layer.weight_scale_inv, s + ) + @unittest.skipUnless( get_device_sm() == 90, "SM90 UE8M0 activation-scale regression" )