[Perf] mlp: Fuse dense SwiGLU with block-FP8 quantization on decode (#45219)
Co-authored-by: Jayden Yu <jaydenyu@fb.com> Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
co-authored by
Jayden Yu
Xiaoyu Zhang
parent
5849ff5337
commit
bb9e9d5632
@@ -238,7 +238,8 @@ template <
|
||||
bool kUe8m0_,
|
||||
bool kRowMajor_,
|
||||
bool kAligned_,
|
||||
bool kFuseSiluAndMul_>
|
||||
bool kFuseSiluAndMul_,
|
||||
bool kDenseSiluAndMul = false>
|
||||
struct QuantTrait {
|
||||
// rename
|
||||
using InputType = InputType_;
|
||||
@@ -253,6 +254,7 @@ struct QuantTrait {
|
||||
static constexpr uint32_t kNumLanes = kGroupSize / kVecSize;
|
||||
static_assert(sizeof(InputType) == 2 || sizeof(InputType) == 4, "inputs must be 16-bit (bf16/fp16) or fp32");
|
||||
static_assert(sizeof(InputType) == 2 || !kFuseSiluAndMul, "fp32 inputs do not implement the fused silu");
|
||||
static_assert(!kDenseSiluAndMul || (kFuseSiluAndMul && kGroupSize == 128 && std::is_same_v<QuantType, fp8_e4m3_t>));
|
||||
static_assert(16 <= kGroupSize && kGroupSize <= 256, "supported group sizes are 16..256");
|
||||
static_assert(kGroupSize % kVecSize == 0 && 1 <= kNumLanes && kNumLanes <= device::kWarpThreads);
|
||||
static_assert(!kUe8m0 || std::is_same_v<QuantType, fp8_e4m3_t>, "ue8m0 scales imply fp8 output");
|
||||
@@ -302,8 +304,15 @@ struct QuantTrait {
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kVecSize / 2; ++i) {
|
||||
const auto gate = cast<float2>(in[i]);
|
||||
const auto act = cast<T2>(float2{detail::silu(gate.x), detail::silu(gate.y)});
|
||||
in[i] = __hmul2(act, up[i]);
|
||||
if constexpr (kDenseSiluAndMul) {
|
||||
// Match dense activation.cuh: round the product, not SiLU, to T.
|
||||
const auto up_f32 = cast<float2>(up[i]);
|
||||
in[i] = cast<T2>(
|
||||
float2{(gate.x / (1.0f + expf(-gate.x))) * up_f32.x, (gate.y / (1.0f + expf(-gate.y))) * up_f32.y});
|
||||
} else {
|
||||
const auto act = cast<T2>(float2{detail::silu(gate.x), detail::silu(gate.y)});
|
||||
in[i] = __hmul2(act, up[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -342,7 +351,9 @@ struct QuantTrait {
|
||||
} else {
|
||||
// fp32 scale: multiply in fp32 (hmul2 brings too much precision loss)
|
||||
scale_inv = raw_scale;
|
||||
const float quant_scale = kMaxValue / amax;
|
||||
// Dense mode uses activation's compiler flags, but quantization must
|
||||
// retain the fast division used by the standalone quantizer.
|
||||
const float quant_scale = kDenseSiluAndMul ? __fdividef(kMaxValue, amax) : kMaxValue / amax;
|
||||
const float2 quant_scale2 = {quant_scale, quant_scale};
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kVecSize / 2; ++i) {
|
||||
@@ -621,9 +632,11 @@ template <
|
||||
bool kRowMajor,
|
||||
bool kAligned,
|
||||
bool kFuseSiluAndMul,
|
||||
bool kUsePDL>
|
||||
bool kUsePDL,
|
||||
bool kDenseSiluAndMul = false>
|
||||
struct PerTokenGroupQuantFlatKernel {
|
||||
using Trait = QuantTrait<InputType, QuantType, kGroupSize, kUe8m0, kRowMajor, kAligned, kFuseSiluAndMul>;
|
||||
using Trait =
|
||||
QuantTrait<InputType, QuantType, kGroupSize, kUe8m0, kRowMajor, kAligned, kFuseSiluAndMul, kDenseSiluAndMul>;
|
||||
|
||||
static void run(tvm::ffi::TensorView input, tvm::ffi::TensorView output_q, tvm::ffi::TensorView output_s) {
|
||||
const auto ctx = build_quant_context<Trait, /*kMasked=*/false>(input, output_q, output_s);
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Optional, Tuple
|
||||
from typing import TYPE_CHECKING, Literal, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import (
|
||||
cache_once,
|
||||
get_activation_cuda_cflags,
|
||||
is_arch_support_pdl,
|
||||
load_jit,
|
||||
make_cpp_args,
|
||||
@@ -33,10 +34,26 @@ def _jit_module(
|
||||
masked_layout: bool,
|
||||
use_pdl: bool,
|
||||
sorted_layout: bool = False,
|
||||
silu_and_mul_mode: str = "legacy",
|
||||
) -> Module:
|
||||
assert in_dtype in _SUPPORTED_INPUT_DTYPES
|
||||
assert out_dtype in _SUPPORTED_OUTPUT_DTYPES
|
||||
assert group_size in _SUPPORTED_GROUP_SIZES
|
||||
if silu_and_mul_mode not in ("legacy", "dense"):
|
||||
raise ValueError(f"Unknown silu_and_mul_mode: {silu_and_mul_mode}")
|
||||
dense = silu_and_mul_mode == "dense"
|
||||
if dense and not (
|
||||
fuse_silu_and_mul
|
||||
and in_dtype in (torch.bfloat16, torch.float16)
|
||||
and out_dtype == torch.float8_e4m3fn
|
||||
and group_size == 128
|
||||
and not masked_layout
|
||||
and not sorted_layout
|
||||
):
|
||||
raise ValueError(
|
||||
"dense mode requires flat BF16/FP16 SiLU+mul, FP8 output and group_size=128"
|
||||
)
|
||||
# Only the flat dense variant adds a flag; legacy launcher arguments stay intact.
|
||||
trait_args = make_cpp_args(
|
||||
in_dtype,
|
||||
out_dtype,
|
||||
@@ -46,6 +63,7 @@ def _jit_module(
|
||||
aligned,
|
||||
fuse_silu_and_mul,
|
||||
use_pdl,
|
||||
*([True] if dense else []),
|
||||
)
|
||||
assert not (masked_layout and sorted_layout)
|
||||
layout = "masked" if masked_layout else ("sorted" if sorted_layout else "flat")
|
||||
@@ -56,7 +74,9 @@ def _jit_module(
|
||||
layout,
|
||||
cuda_files=["gemm/per_token_group_quant.cuh"],
|
||||
cuda_wrappers=[("per_token_group_quant", f"{launcher}<{trait_args}>::run")],
|
||||
extra_cuda_cflags=["--use_fast_math"],
|
||||
extra_cuda_cflags=get_activation_cuda_cflags()
|
||||
if dense
|
||||
else ["--use_fast_math"],
|
||||
)
|
||||
|
||||
|
||||
@@ -97,6 +117,7 @@ def _per_token_group_quant_custom_op(
|
||||
sorted_token_ids: Optional[torch.Tensor] = None,
|
||||
num_tokens_post_padded: Optional[torch.Tensor] = None,
|
||||
num_valid_tokens: int = 0,
|
||||
silu_and_mul_mode: str = "legacy",
|
||||
) -> None:
|
||||
num_groups = output_q.shape[-1] // group_size
|
||||
row_major, aligned = _infer_scale_layout(output_s, scale_ue8m0, num_groups)
|
||||
@@ -111,6 +132,7 @@ def _per_token_group_quant_custom_op(
|
||||
masked_m is not None,
|
||||
is_arch_support_pdl(),
|
||||
sorted_token_ids is not None,
|
||||
silu_and_mul_mode,
|
||||
)
|
||||
if masked_m is not None:
|
||||
module.per_token_group_quant(
|
||||
@@ -186,6 +208,7 @@ def per_token_group_quant(
|
||||
sorted_token_ids: Optional[torch.Tensor] = None,
|
||||
num_tokens_post_padded: Optional[torch.Tensor] = None,
|
||||
num_valid_tokens: int = 0,
|
||||
silu_and_mul_mode: Literal["legacy", "dense"] = "legacy",
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Per-token-group quantization. Returns ``(output_q, output_s)``.
|
||||
|
||||
@@ -207,6 +230,11 @@ def per_token_group_quant(
|
||||
int32 contiguous -> row-major UE8M0 bytes packed 4-per-int32
|
||||
The packed layouts require ``scale_ue8m0=True``.
|
||||
|
||||
With ``fuse_silu_and_mul=True``, ``silu_and_mul_mode="dense"`` matches the
|
||||
dense CUDA activation: FP32 SiLU and multiply, then one source-dtype round
|
||||
before quantizing. The default retains the existing early SiLU rounding.
|
||||
Dense mode supports flat BF16/FP16 inputs, FP8 output and groups of 128.
|
||||
|
||||
``expected_m`` (masked only) is an optional expected-tokens-per-expert hint.
|
||||
|
||||
``sorted_token_ids`` selects the sorted layout: ``input`` rows follow
|
||||
@@ -242,5 +270,6 @@ def per_token_group_quant(
|
||||
sorted_token_ids=sorted_token_ids,
|
||||
num_tokens_post_padded=num_tokens_post_padded,
|
||||
num_valid_tokens=num_valid_tokens,
|
||||
silu_and_mul_mode=silu_and_mul_mode,
|
||||
)
|
||||
return output_q, output_s
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Dense activation/quantization fusion for the block-FP8 DeepGEMM path."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.fused_op import get_fused_op_backend
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_forward,
|
||||
get_lora,
|
||||
get_parallel,
|
||||
get_platform,
|
||||
)
|
||||
from sglang.srt.utils import is_cuda
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.linear import RowParallelLinear
|
||||
|
||||
|
||||
class DenseSiluAndMulQuant:
|
||||
"""Keep the normal activation unless down_proj can consume our FP8 tuple."""
|
||||
|
||||
def __init__(self, down_proj: RowParallelLinear) -> None:
|
||||
self.enabled = False
|
||||
self.down_proj = down_proj
|
||||
self.quant_method = down_proj.quant_method
|
||||
if not (is_cuda() and get_platform().is_sm100 and get_parallel().tp_size == 1):
|
||||
return
|
||||
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.linear import RowParallelLinear
|
||||
from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod
|
||||
from sglang.srt.layers.quantization.fp8_utils import (
|
||||
_deepgemm_block_fp8_supported,
|
||||
deepgemm_w8a8_block_fp8_linear_with_fallback,
|
||||
)
|
||||
|
||||
method = self.quant_method
|
||||
if not (
|
||||
type(down_proj) is RowParallelLinear
|
||||
and type(method) is Fp8LinearMethod
|
||||
and method.weight_block_size == [128, 128]
|
||||
and method.quant_config.activation_scheme == "dynamic"
|
||||
and not (method.use_marlin or method.use_mxfp8 or method.block_fp8_as_mxfp8)
|
||||
and deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
|
||||
and down_proj.input_is_parallel
|
||||
):
|
||||
return
|
||||
backend = method.w8a8_block_fp8_linear
|
||||
if isinstance(backend, partial):
|
||||
if backend.args or set(backend.keywords) - {"act_scale_ue8m0"}:
|
||||
return
|
||||
backend = backend.func
|
||||
# The supported loader preserves matrix geometry, not tensor identity/device.
|
||||
self.weight_shape = (down_proj.output_size, down_proj.input_size_per_partition)
|
||||
self.enabled = (
|
||||
backend is deepgemm_w8a8_block_fp8_linear_with_fallback
|
||||
and _deepgemm_block_fp8_supported(
|
||||
self.weight_shape, method.weight_block_size, torch.bfloat16
|
||||
)
|
||||
)
|
||||
|
||||
def forward_decode(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
act_fn: SiluAndMul,
|
||||
down_proj: RowParallelLinear,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Try fusion for decode; callers must bypass this helper in other modes."""
|
||||
if not self.enabled or torch.compiler.is_compiling():
|
||||
return act_fn(x)
|
||||
|
||||
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
|
||||
deterministic = get_exec().deterministic
|
||||
if (
|
||||
down_proj is not self.down_proj
|
||||
or down_proj.quant_method is not self.quant_method
|
||||
or down_proj.weight.shape != self.weight_shape
|
||||
or type(act_fn) is not SiluAndMul
|
||||
or (act_fn._forward_method or act_fn._resolve_forward_method())
|
||||
!= act_fn.forward_cuda
|
||||
or act_fn.is_torch_compile
|
||||
or get_fused_op_backend() is not None
|
||||
or deterministic.enable_deterministic_inference
|
||||
or deterministic.rl_on_policy_target is not None
|
||||
or is_batch_invariant_mode_enabled()
|
||||
or get_lora().enable_lora
|
||||
or get_forward().sp_active
|
||||
or x.dtype != torch.bfloat16
|
||||
or not x.is_cuda
|
||||
or x.ndim != 2
|
||||
or x.shape[0] == 0
|
||||
or not x.is_contiguous()
|
||||
or x.data_ptr() % 32 != 0
|
||||
or x.stride(0) % 16 != 0
|
||||
or x.device != down_proj.weight.device
|
||||
or x.shape[1] != 2 * self.weight_shape[1]
|
||||
):
|
||||
return act_fn(x)
|
||||
|
||||
from sglang.kernels.ops.quantization.per_token_group_quant import (
|
||||
per_token_group_quant,
|
||||
)
|
||||
|
||||
return per_token_group_quant(
|
||||
x,
|
||||
group_size=128,
|
||||
scale_ue8m0=True,
|
||||
column_major_scales=True,
|
||||
fuse_silu_and_mul=True,
|
||||
silu_and_mul_mode="dense",
|
||||
)
|
||||
@@ -1453,6 +1453,25 @@ def _deepgemm_block_fp8_supported(
|
||||
return list(block_size) == [128, 128] and n % 64 == 0 and k % 128 == 0
|
||||
|
||||
|
||||
def _deepgemm_packed_input_valid(input, weight, input_scale, weight_scale) -> bool:
|
||||
"""Validate the group-128, column-major TMA layout for packed activations."""
|
||||
if input.ndim != 2 or weight.ndim != 2:
|
||||
return False
|
||||
m, k = input.shape
|
||||
packed_groups = ceil_div(k // 128, 4)
|
||||
padded_m = ceil_align(m, 4)
|
||||
return (
|
||||
input.is_cuda
|
||||
and input.is_contiguous()
|
||||
and k == weight.shape[1]
|
||||
and input.device == weight.device == input_scale.device == weight_scale.device
|
||||
and input_scale.shape == (m, packed_groups)
|
||||
and input_scale.stride() == (1, padded_m)
|
||||
and input.data_ptr() % 16 == 0
|
||||
and input_scale.data_ptr() % 16 == 0
|
||||
)
|
||||
|
||||
|
||||
def deepgemm_w8a8_block_fp8_linear_with_fallback(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
@@ -1463,18 +1482,36 @@ def deepgemm_w8a8_block_fp8_linear_with_fallback(
|
||||
act_scale_ue8m0: bool = False,
|
||||
weight_bf16: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
if input_scale is not None:
|
||||
# Block32 pre-quantized input reaches Triton below on UE8M0 DeepGEMM.
|
||||
assert (
|
||||
list(block_size) == [32, 32] or not deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
|
||||
)
|
||||
assert input.dtype == torch.float8_e4m3fn
|
||||
output_dtype = torch.bfloat16 if input_scale is not None else input.dtype
|
||||
|
||||
# TODO: https://github.com/sgl-project/sglang/pull/6890#issuecomment-2943395737
|
||||
if not _deepgemm_block_fp8_supported(
|
||||
deepgemm_supported = weight.ndim == 2 and _deepgemm_block_fp8_supported(
|
||||
weight.shape, block_size, output_dtype
|
||||
) or _ue8m0_deepgemm_skips_block32(block_size, weight_scale, input_scale):
|
||||
)
|
||||
if input_scale is not None:
|
||||
if input_scale.dtype == torch.int32:
|
||||
# Packed activation scales cannot go through the Triton fallback.
|
||||
if not (
|
||||
deepgemm_supported
|
||||
and deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
|
||||
and list(block_size) == [128, 128]
|
||||
and _deepgemm_packed_input_valid(
|
||||
input, weight, input_scale, weight_scale
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
"Pre-quantized DeepGEMM block-FP8 input requires supported "
|
||||
"[128, 128] weights and TMA-aligned, column-major packed UE8M0 scales"
|
||||
)
|
||||
else:
|
||||
# Block32 pre-quantized input uses the Triton fallback.
|
||||
assert (
|
||||
list(block_size) == [32, 32]
|
||||
or not deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
|
||||
)
|
||||
assert input.dtype == torch.float8_e4m3fn
|
||||
# TODO: https://github.com/sgl-project/sglang/pull/6890#issuecomment-2943395737
|
||||
if not deepgemm_supported or _ue8m0_deepgemm_skips_block32(
|
||||
block_size, weight_scale, input_scale
|
||||
):
|
||||
# fall back to triton
|
||||
# If weight_scale is in UE8M0 packed format (int32), convert back to float32
|
||||
# UE8M0 format has shape (N, K//block_k//4) with dtype int32
|
||||
@@ -1495,8 +1532,8 @@ def deepgemm_w8a8_block_fp8_linear_with_fallback(
|
||||
)
|
||||
|
||||
if input_scale is not None:
|
||||
# Pre-quantized activations retain their per-token group size and
|
||||
# fp32 scales. DeepGEMM aligns scale storage for TMA when needed.
|
||||
# Pre-quantized activations retain their group size and scale format.
|
||||
# DeepGEMM aligns fp32 scale storage for TMA when needed.
|
||||
input_2d = input.view(-1, input.shape[-1])
|
||||
output = w8a8_block_fp8_matmul_deepgemm(
|
||||
input_2d,
|
||||
|
||||
@@ -49,6 +49,7 @@ from sglang.srt.layers.linear import (
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
|
||||
from sglang.srt.layers.pooler import Pooler, PoolingType
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.fp8_activation import DenseSiluAndMulQuant
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
|
||||
@@ -118,6 +119,7 @@ class LlamaMLP(nn.Module):
|
||||
)
|
||||
self.act_fn = SiluAndMul()
|
||||
self.dual_gemm = DualGemm(self.gate_up_proj, self.down_proj, hidden_size)
|
||||
self.activation_quant = DenseSiluAndMulQuant(self.down_proj)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -128,7 +130,12 @@ class LlamaMLP(nn.Module):
|
||||
x = self.dual_gemm(x, self.gate_up_proj)
|
||||
else:
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
if forward_batch is not None and forward_batch.forward_mode.is_decode():
|
||||
x = self.activation_quant.forward_decode(
|
||||
gate_up, self.act_fn, self.down_proj
|
||||
)
|
||||
else:
|
||||
x = self.act_fn(gate_up)
|
||||
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
@@ -404,7 +411,7 @@ class LlamaDecoderLayer(nn.Module):
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
|
||||
@@ -46,6 +46,7 @@ from sglang.srt.layers.linear import (
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.pooler import Pooler, PoolingType
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.fp8_activation import DenseSiluAndMulQuant
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
|
||||
@@ -102,6 +103,7 @@ class Qwen2MLP(nn.Module):
|
||||
)
|
||||
self.act_fn = SiluAndMul()
|
||||
self.dual_gemm = DualGemm(self.gate_up_proj, self.down_proj, hidden_size)
|
||||
self.activation_quant = DenseSiluAndMulQuant(self.down_proj)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -115,7 +117,12 @@ class Qwen2MLP(nn.Module):
|
||||
x = self.dual_gemm(x, self.gate_up_proj)
|
||||
else:
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
if forward_batch is not None and forward_batch.forward_mode.is_decode():
|
||||
x = self.activation_quant.forward_decode(
|
||||
gate_up, self.act_fn, self.down_proj
|
||||
)
|
||||
else:
|
||||
x = self.act_fn(gate_up)
|
||||
x, _ = self.down_proj(x, forward_batch=forward_batch)
|
||||
return x
|
||||
|
||||
@@ -339,7 +346,7 @@ class Qwen2DecoderLayer(nn.Module):
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Paired B200 MLP timing for dense SwiGLU fusion.
|
||||
|
||||
Synthetic weights test the operator pipeline, not model quality or serving TBT.
|
||||
Operator parity and the opt-in numerical sweep live in test_per_token_group_quant.py.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import random
|
||||
import statistics
|
||||
import subprocess
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.benchmark import marker
|
||||
from sglang.kernels.ops.activation.activation import silu_and_mul
|
||||
from sglang.kernels.ops.quantization.per_token_group_quant import per_token_group_quant
|
||||
|
||||
|
||||
def mlp(x, gate_up, gate_scale, down, down_scale, *, fused):
|
||||
from sglang.srt.layers.quantization.fp8_utils import (
|
||||
deepgemm_w8a8_block_fp8_linear_with_fallback as linear,
|
||||
)
|
||||
|
||||
projected = linear(x, gate_up, [128, 128], gate_scale)
|
||||
if fused:
|
||||
q, s = per_token_group_quant(
|
||||
projected,
|
||||
scale_ue8m0=True,
|
||||
column_major_scales=True,
|
||||
fuse_silu_and_mul=True,
|
||||
silu_and_mul_mode="dense",
|
||||
)
|
||||
return linear(q, down, [128, 128], down_scale, s)
|
||||
return linear(silu_and_mul(projected), down, [128, 128], down_scale)
|
||||
|
||||
|
||||
def benchmark(args):
|
||||
from functools import partial
|
||||
|
||||
torch.manual_seed(20261017)
|
||||
order = random.Random(17)
|
||||
for m in args.batches:
|
||||
k, h = args.intermediate_size, 4096
|
||||
assert k % 512 == 0
|
||||
x = torch.randn(m, h, device="cuda", dtype=torch.bfloat16)
|
||||
up = (torch.randn(2 * k, h, device="cuda") * 0.01).to(torch.float8_e4m3fn)
|
||||
down = (torch.randn(h, k, device="cuda") * 0.01).to(torch.float8_e4m3fn)
|
||||
# Packed exponent 127 represents scale=1, using DeepGEMM's weight layout.
|
||||
us = torch.full(
|
||||
(h // 512, 2 * k), 0x7F7F7F7F, device="cuda", dtype=torch.int32
|
||||
).T
|
||||
ds = torch.full((k // 512, h), 0x7F7F7F7F, device="cuda", dtype=torch.int32).T
|
||||
inputs = (x, up, us, down, ds)
|
||||
variants = {
|
||||
"eager": partial(mlp, fused=False),
|
||||
"fused": partial(mlp, fused=True),
|
||||
"control": partial(mlp, fused=False),
|
||||
}
|
||||
torch.testing.assert_close(
|
||||
variants["eager"](*inputs), variants["fused"](*inputs), rtol=0, atol=0
|
||||
)
|
||||
if args.profile:
|
||||
for name in ("eager", "fused"):
|
||||
with torch.profiler.profile(
|
||||
activities=[
|
||||
torch.profiler.ProfilerActivity.CPU,
|
||||
torch.profiler.ProfilerActivity.CUDA,
|
||||
]
|
||||
) as prof:
|
||||
variants[name](*inputs)
|
||||
torch.cuda.synchronize()
|
||||
kernels = [
|
||||
e.name
|
||||
for e in prof.events()
|
||||
if e.device_type == torch.autograd.DeviceType.CUDA
|
||||
]
|
||||
print(
|
||||
json.dumps({"batch": m, "variant": name, "CUDA_events": kernels}),
|
||||
flush=True,
|
||||
)
|
||||
samples = {name: [] for name in variants}
|
||||
for _ in range(args.rounds):
|
||||
names = list(variants)
|
||||
order.shuffle(names)
|
||||
for name in names:
|
||||
# Rotate all inputs, including both GEMM weights, beyond L2.
|
||||
result = marker.do_bench(
|
||||
variants[name],
|
||||
input_args=inputs,
|
||||
metrics=(0.5,),
|
||||
estimated_time_ms=30,
|
||||
warmup_iters=3,
|
||||
graph_clone_args="all",
|
||||
disable_log_bandwidth=True,
|
||||
)
|
||||
samples[name].append(result.times[0] * 1e6)
|
||||
eager, fused, control = (samples[n] for n in ("eager", "fused", "control"))
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"batch": m,
|
||||
"intermediate_size": k,
|
||||
"rounds": args.rounds,
|
||||
"cold_L2": True,
|
||||
"raw_us": samples,
|
||||
"median_us": {n: statistics.median(s) for n, s in samples.items()},
|
||||
"paired_wins": sum(a > b for a, b in zip(eager, fused)),
|
||||
"paired_saving_us": statistics.median(
|
||||
a - b for a, b in zip(eager, fused)
|
||||
),
|
||||
"control_adjusted_saving_us": statistics.median(
|
||||
(a + c) / 2 - b for a, b, c in zip(eager, fused, control)
|
||||
),
|
||||
"AA_median_abs_difference_us": statistics.median(
|
||||
abs(a - c) for a, c in zip(eager, control)
|
||||
),
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--rounds", type=int, default=10)
|
||||
parser.add_argument("--batches", type=int, nargs="+", default=[1, 32, 128])
|
||||
parser.add_argument("--intermediate-size", type=int, default=14336)
|
||||
parser.add_argument("--profile", action="store_true")
|
||||
args = parser.parse_args()
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"base_commit": subprocess.check_output(
|
||||
["git", "rev-parse", "HEAD"], text=True
|
||||
).strip(),
|
||||
"worktree_dirty": bool(
|
||||
subprocess.check_output(["git", "status", "--porcelain"], text=True)
|
||||
),
|
||||
"torch": torch.__version__,
|
||||
"gpu": torch.cuda.get_device_name(),
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
with torch.inference_mode():
|
||||
benchmark(args)
|
||||
@@ -1,5 +1,6 @@
|
||||
from sglang.kernels.jit.benchmark import marker
|
||||
from sglang.kernels.jit.benchmark.utils import create_empty, create_random
|
||||
from sglang.kernels.ops.activation.activation import silu_and_mul
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import (
|
||||
create_per_token_group_quant_fp8_output_scale,
|
||||
fp8_dtype,
|
||||
@@ -18,7 +19,7 @@ from sglang.kernels.ops.quantization.per_token_group_quant_8bit_v2 import (
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(
|
||||
est_time=25, stage="base-b-kernel-benchmark", runner_config="1-gpu-large"
|
||||
est_time=35, stage="base-b-kernel-benchmark", runner_config="1-gpu-large"
|
||||
)
|
||||
|
||||
HIDDEN = 2048
|
||||
@@ -73,5 +74,23 @@ def benchmark(group_size: int, layout: str, num_tokens: int, impl: str):
|
||||
)
|
||||
|
||||
|
||||
def _dense_activation_quant(x, fused):
|
||||
return per_token_group_quant(
|
||||
x if fused else silu_and_mul(x),
|
||||
fuse_silu_and_mul=fused,
|
||||
silu_and_mul_mode="dense" if fused else "legacy",
|
||||
scale_ue8m0=True,
|
||||
column_major_scales=True,
|
||||
)
|
||||
|
||||
|
||||
@marker.parametrize("num_tokens", [1, 32, 128], ci_vals=[1, 32])
|
||||
@marker.benchmark("fused", [False, True])
|
||||
def benchmark_dense(num_tokens: int, fused: bool):
|
||||
x = create_random(num_tokens, 8192)
|
||||
return marker.do_bench(_dense_activation_quant, input_args=(x, fused))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
benchmark.run()
|
||||
benchmark_dense.run()
|
||||
|
||||
@@ -21,7 +21,8 @@ import itertools
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import get_ci_test_range
|
||||
from sglang.kernels.jit.utils import get_ci_test_range, should_run_full_tests
|
||||
from sglang.kernels.ops.activation.activation import silu_and_mul
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import (
|
||||
create_per_token_group_quant_fp8_output_scale,
|
||||
fp8_dtype,
|
||||
@@ -32,9 +33,9 @@ from sglang.kernels.ops.quantization.per_token_group_quant import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=65, stage="base-b-kernel-unit", runner_config="1-gpu-large")
|
||||
register_cuda_ci(est_time=90, stage="base-b-kernel-unit", runner_config="4-gpu-b200")
|
||||
register_cuda_ci(est_time=120, stage="nightly", runner_config="1-gpu-large")
|
||||
register_cuda_ci(est_time=120, stage="base-b-kernel-unit", runner_config="1-gpu-large")
|
||||
register_cuda_ci(est_time=200, stage="base-b-kernel-unit", runner_config="4-gpu-b200")
|
||||
register_cuda_ci(est_time=180, stage="nightly", runner_config="1-gpu-large")
|
||||
|
||||
G = 128
|
||||
FMAX = float(fp8_max) # 448 for e4m3
|
||||
@@ -527,6 +528,124 @@ def test_fused_silu(scale_ue8m0, column_major):
|
||||
assert _dequant_rel_err(x_q, deq_scale, act, G) < 0.05
|
||||
|
||||
|
||||
def _check_dense_silu(x, scale_ue8m0, column_major):
|
||||
original = x.view(torch.int16).clone()
|
||||
kwargs = dict(scale_ue8m0=scale_ue8m0, column_major_scales=column_major)
|
||||
expected_q, expected_s = per_token_group_quant(silu_and_mul(x), **kwargs)
|
||||
q, s = per_token_group_quant(
|
||||
x, fuse_silu_and_mul=True, silu_and_mul_mode="dense", **kwargs
|
||||
)
|
||||
assert torch.equal(q.view(torch.uint8), expected_q.view(torch.uint8))
|
||||
assert torch.equal(s.view(torch.int32), expected_s.view(torch.int32))
|
||||
assert torch.equal(x.view(torch.int16), original)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"dtype,scale_ue8m0,column_major",
|
||||
get_ci_test_range(
|
||||
list(
|
||||
itertools.product(
|
||||
[torch.bfloat16, torch.float16], [False, True], [False, True]
|
||||
)
|
||||
),
|
||||
[
|
||||
(torch.bfloat16, True, True),
|
||||
(torch.float16, True, True),
|
||||
(torch.bfloat16, False, False),
|
||||
],
|
||||
),
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"shape",
|
||||
get_ci_test_range(
|
||||
[(1, 768), (3, 4096), (32, 12288), (32, 14336), (128, 28672)],
|
||||
[(1, 768), (32, 14336)],
|
||||
),
|
||||
)
|
||||
def test_dense_silu_bitexact(dtype, shape, scale_ue8m0, column_major):
|
||||
"""Dense fusion preserves the separately stored activation's rounding.
|
||||
|
||||
The legacy fused mode rounds SiLU before multiplication and fails this
|
||||
comparison. Use the real CUDA activation as the compiler-math reference.
|
||||
"""
|
||||
torch.manual_seed(42)
|
||||
m, h = shape
|
||||
x = torch.randn(m, 2 * h, device="cuda", dtype=dtype) * 3
|
||||
x[0, :8] = torch.tensor(
|
||||
[0.0, -0.0, 1e-6, -1e-6, 100, -100, float("inf"), float("nan")],
|
||||
device="cuda",
|
||||
dtype=dtype,
|
||||
)
|
||||
_check_dense_silu(x, scale_ue8m0, column_major)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not should_run_full_tests(),
|
||||
reason="Set SGLANG_JIT_KERNEL_RUN_FULL_TESTS=1 for the sweep",
|
||||
)
|
||||
def test_dense_silu_extended():
|
||||
"""Opt-in amplitude/layout sweep and all 65,536 BF16/FP16 gate encodings."""
|
||||
torch.manual_seed(20261017)
|
||||
layouts = list(itertools.product((False, True), repeat=2))
|
||||
for dtype in (torch.bfloat16, torch.float16):
|
||||
for (m, k), amplitude in itertools.product(
|
||||
((1, 768), (3, 4096), (32, 12288), (32, 14336), (128, 28672)),
|
||||
(0, 0.001, 1, 10, 100),
|
||||
):
|
||||
x = (torch.randn(m, 2 * k, device="cuda") * amplitude).to(dtype)
|
||||
for ue8m0, column in layouts:
|
||||
_check_dense_silu(x, ue8m0, column)
|
||||
# Signed zeros, subnormals, finite extremes, infinities and NaNs.
|
||||
gate = (
|
||||
torch.arange(65536, device="cuda", dtype=torch.int32)
|
||||
.to(torch.int16)
|
||||
.view(dtype)
|
||||
.reshape(64, 1024)
|
||||
)
|
||||
for up_value in (0, 0.25, 1, 1.125, -2.25):
|
||||
x = torch.cat((gate, torch.full_like(gate, up_value)), dim=1)
|
||||
for ue8m0, column in layouts:
|
||||
_check_dense_silu(x, ue8m0, column)
|
||||
|
||||
|
||||
def test_dense_silu_graph_replay():
|
||||
"""Replays must use new input values and retain the allocated output buffers."""
|
||||
x = torch.randn(3, 1536, device="cuda", dtype=torch.bfloat16)
|
||||
kwargs = dict(scale_ue8m0=True, column_major_scales=True)
|
||||
q, s = per_token_group_quant(
|
||||
x, fuse_silu_and_mul=True, silu_and_mul_mode="dense", **kwargs
|
||||
)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
per_token_group_quant(
|
||||
x, q, s, fuse_silu_and_mul=True, silu_and_mul_mode="dense", **kwargs
|
||||
)
|
||||
for amplitude in (0, 0.001, 10):
|
||||
x.normal_().mul_(amplitude)
|
||||
graph.replay()
|
||||
expected_q, expected_s = per_token_group_quant(silu_and_mul(x), **kwargs)
|
||||
assert torch.equal(q.view(torch.uint8), expected_q.view(torch.uint8))
|
||||
assert torch.equal(s, expected_s)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"overrides",
|
||||
[
|
||||
{"silu_and_mul_mode": "unknown"},
|
||||
{"fuse_silu_and_mul": False},
|
||||
{"out_dtype": torch.int8},
|
||||
{"group_size": 64},
|
||||
{"masked_m": torch.tensor([1], dtype=torch.int32)},
|
||||
],
|
||||
)
|
||||
def test_dense_silu_invalid_mode(overrides):
|
||||
x = torch.zeros(1, 256, device="cuda", dtype=torch.bfloat16)
|
||||
kwargs = dict(fuse_silu_and_mul=True, silu_and_mul_mode="dense")
|
||||
kwargs.update(overrides)
|
||||
with pytest.raises(ValueError, match="mode"):
|
||||
per_token_group_quant(x, **kwargs)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Masked EP-MoE schedule.
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
@@ -0,0 +1,312 @@
|
||||
"""Dense SwiGLU/block-FP8 integration: real MLPs, dispatch and graph replay."""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.quantization import fp8_utils
|
||||
from sglang.srt.layers.quantization.fp8 import Fp8Config
|
||||
from sglang.srt.layers.quantization.fp8_utils import Fp8GemmRunnerBackend
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.model_loader.utils import set_default_torch_dtype
|
||||
from sglang.srt.utils import get_device_sm
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.layer_ut_utils import init_single_process_dist
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=45, stage="base-b", runner_config="4-gpu-b200")
|
||||
|
||||
|
||||
@unittest.skipUnless(get_device_sm() in (100, 103), "Dense block-FP8 fusion on SM100")
|
||||
@mock.patch.object(fp8_utils, "FP8_GEMM_RUNNER_BACKEND", Fp8GemmRunnerBackend.DEEP_GEMM)
|
||||
class TestDenseSiluAndMulQuant(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
init_single_process_dist()
|
||||
|
||||
def _make_mlp(self, model):
|
||||
from sglang.srt.models.llama import LlamaMLP
|
||||
from sglang.srt.models.qwen2 import Qwen2MLP
|
||||
|
||||
config = Fp8Config(
|
||||
is_checkpoint_fp8_serialized=True,
|
||||
activation_scheme="dynamic",
|
||||
weight_block_size=[128, 128],
|
||||
)
|
||||
with set_default_torch_dtype(torch.bfloat16):
|
||||
mlp = {"llama": LlamaMLP, "qwen": Qwen2MLP}[model](
|
||||
512,
|
||||
768,
|
||||
"silu",
|
||||
quant_config=config,
|
||||
reduce_results=False,
|
||||
).cuda()
|
||||
with torch.no_grad():
|
||||
for layer in (mlp.gate_up_proj, mlp.down_proj):
|
||||
weight = (
|
||||
torch.randn(layer.weight.shape, device="cuda", dtype=torch.bfloat16)
|
||||
* 0.02
|
||||
)
|
||||
layer.weight.copy_(weight.to(torch.float8_e4m3fn))
|
||||
layer.weight_scale_inv.fill_(1)
|
||||
layer.quant_method.process_weights_after_loading(layer)
|
||||
return mlp
|
||||
|
||||
def test_mlp_parity_and_graph_replay(self):
|
||||
"""Cold first-decode dispatch and graph replay must match the unfused MLP."""
|
||||
batch = SimpleNamespace(forward_mode=ForwardMode.DECODE)
|
||||
torch.manual_seed(123)
|
||||
with torch.inference_mode():
|
||||
for model in ("llama", "qwen"):
|
||||
mlp = self._make_mlp(model)
|
||||
self.assertTrue(mlp.activation_quant.enabled)
|
||||
self.assertIsNone(mlp.dual_gemm.mode)
|
||||
self.assertIsNone(mlp.act_fn._forward_method)
|
||||
for m in (1, 3, 32, 128):
|
||||
with self.subTest(model=model, m=m):
|
||||
x = torch.randn(m, 512, device="cuda", dtype=torch.bfloat16)
|
||||
gate_up, _ = mlp.gate_up_proj(x)
|
||||
quantized = mlp.activation_quant.forward_decode(
|
||||
gate_up, mlp.act_fn, mlp.down_proj
|
||||
)
|
||||
self.assertIsInstance(quantized, tuple)
|
||||
expected = mlp(x)
|
||||
actual = mlp(x, batch)
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
actual = mlp(x, batch)
|
||||
for amplitude in (0, 0.1, 3):
|
||||
x.normal_().mul_(amplitude)
|
||||
graph.replay()
|
||||
torch.testing.assert_close(actual, mlp(x), rtol=0, atol=0)
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_non_decode_bypass(self):
|
||||
"""Every non-decode mode must avoid the helper and preserve MLP output."""
|
||||
for model in ("llama", "qwen"):
|
||||
mlp = self._make_mlp(model)
|
||||
x = torch.randn(1, 512, device="cuda", dtype=torch.bfloat16)
|
||||
expected = mlp(x)
|
||||
with mock.patch.object(
|
||||
type(mlp.activation_quant),
|
||||
"forward_decode",
|
||||
side_effect=AssertionError("non-decode entered fusion dispatch"),
|
||||
):
|
||||
for mode in (None, *ForwardMode):
|
||||
if mode == ForwardMode.DECODE:
|
||||
continue
|
||||
with self.subTest(model=model, mode=mode):
|
||||
batch = (
|
||||
SimpleNamespace(forward_mode=mode)
|
||||
if mode is not None
|
||||
else None
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
mlp(x, batch), expected, rtol=0, atol=0
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_decoder_handoff(self):
|
||||
"""Dropping forward_batch between the decoder and real MLP must fail."""
|
||||
from sglang.srt.models.llama import LlamaDecoderLayer
|
||||
from sglang.srt.models.qwen2 import Qwen2DecoderLayer
|
||||
|
||||
batch = SimpleNamespace(forward_mode=ForwardMode.DECODE)
|
||||
boundary = SimpleNamespace(
|
||||
prepare=lambda x, batch, **kw: x, finish=lambda x, batch: x
|
||||
)
|
||||
for model, decoder_cls in (
|
||||
("llama", LlamaDecoderLayer),
|
||||
("qwen", Qwen2DecoderLayer),
|
||||
):
|
||||
mlp = self._make_mlp(model)
|
||||
x = torch.randn(1, 512, device="cuda", dtype=torch.bfloat16)
|
||||
expected = mlp(x)
|
||||
# Only attention/communication are stubbed; run the real decoder and MLP.
|
||||
decoder = SimpleNamespace(
|
||||
attn_boundary=boundary,
|
||||
ffn_boundary=boundary,
|
||||
self_attn=lambda **kw: kw["hidden_states"],
|
||||
mlp=mlp,
|
||||
)
|
||||
with (
|
||||
self.subTest(model=model),
|
||||
mock.patch.object(
|
||||
mlp.act_fn,
|
||||
"forward",
|
||||
side_effect=AssertionError("decode used separate activation"),
|
||||
),
|
||||
):
|
||||
actual = decoder_cls.forward(decoder, None, x, batch)
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
|
||||
def test_decode_fallback_overrides_and_inputs(self):
|
||||
"""Overrides and unsupported inputs must retain the original activation."""
|
||||
from sglang.kernels.fused_op import get_fused_op_backend, set_fused_op_backend
|
||||
from sglang.kernels.spec import KernelBackend
|
||||
from sglang.srt.runtime_context import get_exec, get_forward, get_lora
|
||||
|
||||
mlp = self._make_mlp("llama")
|
||||
x = torch.randn(3, 1536, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
def check(*, value=x, projection=mlp.down_proj):
|
||||
actual = mlp.activation_quant.forward_decode(value, mlp.act_fn, projection)
|
||||
self.assertIsInstance(actual, torch.Tensor)
|
||||
torch.testing.assert_close(actual, mlp.act_fn(value), rtol=0, atol=0)
|
||||
|
||||
with get_forward().scoped(sp_active=True):
|
||||
check()
|
||||
with get_lora().override(enable_lora=True):
|
||||
check()
|
||||
with get_exec().deterministic.override(enable_deterministic_inference=True):
|
||||
check()
|
||||
with mock.patch.object(
|
||||
mlp.act_fn, "_forward_method", mlp.act_fn.forward_native
|
||||
):
|
||||
check()
|
||||
old_backend = get_fused_op_backend()
|
||||
try:
|
||||
set_fused_op_backend(KernelBackend.TORCH)
|
||||
check()
|
||||
finally:
|
||||
set_fused_op_backend(old_backend)
|
||||
for dtype in (torch.float16, torch.float32):
|
||||
check(value=x.to(dtype))
|
||||
# Contiguous single-row views may still have an unaligned row stride.
|
||||
check(value=x[:1].as_strided((1, 1536), (1, 1)))
|
||||
|
||||
wrapped_projection = SimpleNamespace(
|
||||
quant_method=mlp.down_proj.quant_method,
|
||||
weight=mlp.down_proj.weight,
|
||||
)
|
||||
check(projection=wrapped_projection)
|
||||
actual = mlp.activation_quant.forward_decode(
|
||||
x, torch.nn.Identity(), mlp.down_proj
|
||||
)
|
||||
self.assertIs(actual, x)
|
||||
|
||||
mlp.act_fn.enter_torch_compile(x.shape[0])
|
||||
try:
|
||||
check()
|
||||
compiled = torch.compile(
|
||||
lambda value: mlp.activation_quant.forward_decode(
|
||||
value, mlp.act_fn, mlp.down_proj
|
||||
),
|
||||
backend="eager",
|
||||
fullgraph=True,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
compiled(x), mlp.act_fn.forward_native(x), rtol=0, atol=0
|
||||
)
|
||||
finally:
|
||||
mlp.act_fn.leave_torch_compile()
|
||||
|
||||
def test_constructor_gates(self):
|
||||
"""Other quantization/backends must retain activation tensors for down_proj."""
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.linear import RowParallelLinear
|
||||
from sglang.srt.layers.quantization.fp8_activation import DenseSiluAndMulQuant
|
||||
from sglang.srt.runtime_context import override_platform
|
||||
from sglang.test.parallel_groups import parallel_scope
|
||||
|
||||
cases = [
|
||||
(None, "deep_gemm"),
|
||||
({}, "deep_gemm"),
|
||||
({"weight_block_size": [128, 128]}, "triton"),
|
||||
({"use_mxfp8": True}, "auto"),
|
||||
({"weight_block_size": [32, 32], "scale_fmt": "ue8m0"}, "auto"),
|
||||
]
|
||||
act = SiluAndMul()
|
||||
x = torch.randn(3, 1536, device="cuda", dtype=torch.bfloat16)
|
||||
expected = act(x)
|
||||
for options, backend in cases:
|
||||
config = (
|
||||
Fp8Config(is_checkpoint_fp8_serialized=True, **options)
|
||||
if options is not None
|
||||
else None
|
||||
)
|
||||
with (
|
||||
self.subTest(config=options, backend=backend),
|
||||
mock.patch.object(
|
||||
fp8_utils, "FP8_GEMM_RUNNER_BACKEND", Fp8GemmRunnerBackend(backend)
|
||||
),
|
||||
):
|
||||
layer = RowParallelLinear(
|
||||
768,
|
||||
512,
|
||||
bias=False,
|
||||
params_dtype=torch.bfloat16,
|
||||
quant_config=config,
|
||||
).cuda()
|
||||
helper = DenseSiluAndMulQuant(layer)
|
||||
torch.testing.assert_close(
|
||||
helper.forward_decode(x, act, layer), expected, rtol=0, atol=0
|
||||
)
|
||||
mlp = self._make_mlp("llama")
|
||||
for scope in (
|
||||
parallel_scope(tp_size=2, attn_tp_size=2, moe_tp_size=2),
|
||||
override_platform(is_sm100=False),
|
||||
):
|
||||
with scope:
|
||||
helper = DenseSiluAndMulQuant(mlp.down_proj)
|
||||
torch.testing.assert_close(
|
||||
helper.forward_decode(x, act, mlp.down_proj), expected, rtol=0, atol=0
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_fixed_geometry_and_live_projection_guards(self):
|
||||
"""Cache support rules, but reject changed geometry/methods and other layers."""
|
||||
from sglang.srt.layers.linear import RowParallelLinear
|
||||
from sglang.srt.layers.quantization.fp8_activation import DenseSiluAndMulQuant
|
||||
|
||||
mlp = self._make_mlp("llama")
|
||||
layer = mlp.down_proj
|
||||
helper = mlp.activation_quant
|
||||
x = torch.randn(3, 1536, device="cuda", dtype=torch.bfloat16)
|
||||
expected = mlp.act_fn(x)
|
||||
# Same-shape weight reloads remain eligible without redoing support checks.
|
||||
with (
|
||||
mock.patch.object(
|
||||
layer, "weight", torch.nn.Parameter(layer.weight.clone(), False)
|
||||
),
|
||||
mock.patch.object(
|
||||
fp8_utils, "_deepgemm_block_fp8_supported", side_effect=AssertionError
|
||||
),
|
||||
):
|
||||
self.assertIsInstance(helper.forward_decode(x, mlp.act_fn, layer), tuple)
|
||||
for attr, value in (
|
||||
("weight", torch.nn.Parameter(layer.weight.reshape(1024, 384), False)),
|
||||
("quant_method", object()),
|
||||
):
|
||||
with self.subTest(attr=attr), mock.patch.object(layer, attr, value):
|
||||
torch.testing.assert_close(
|
||||
helper.forward_decode(x, mlp.act_fn, layer),
|
||||
expected,
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
for n, k in ((512, 768), (513, 768), (512, 769)):
|
||||
other = RowParallelLinear(
|
||||
k,
|
||||
n,
|
||||
bias=False,
|
||||
params_dtype=torch.bfloat16,
|
||||
quant_config=layer.quant_method.quant_config,
|
||||
)
|
||||
with self.subTest(n=n, k=k):
|
||||
self.assertEqual(
|
||||
DenseSiluAndMulQuant(other).enabled, (n, k) == (512, 768)
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
helper.forward_decode(x, mlp.act_fn, other),
|
||||
expected,
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -192,6 +192,46 @@ class TestFp8BlockwiseLinearBackends(_LinearBackendCheck):
|
||||
def test_deep_gemm(self):
|
||||
self._run("deep_gemm")
|
||||
|
||||
@unittest.skipUnless(
|
||||
get_device_sm() in (100, 103), "Packed block-FP8 scales on SM100"
|
||||
)
|
||||
def test_deep_gemm_prequantized(self):
|
||||
"""Packed activations must skip requantization and preserve the GEMM result."""
|
||||
torch.manual_seed(9)
|
||||
with mock.patch.object(
|
||||
fp8_utils, "FP8_GEMM_RUNNER_BACKEND", Fp8GemmRunnerBackend.DEEP_GEMM
|
||||
):
|
||||
for m, n, k in FP8_BLOCK_SHAPES:
|
||||
with self.subTest(shape=(m, n, k)):
|
||||
layer, _ = self._build_layer(n, k)
|
||||
layer.quant_method.process_weights_after_loading(layer)
|
||||
x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16)
|
||||
q, s = fp8_utils.sglang_per_token_group_quant_fp8(
|
||||
x,
|
||||
128,
|
||||
column_major_scales=True,
|
||||
scale_tma_aligned=True,
|
||||
scale_ue8m0=True,
|
||||
)
|
||||
expected, _ = layer(x)
|
||||
actual, _ = layer((q, s))
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
linear = fp8_utils.deepgemm_w8a8_block_fp8_linear_with_fallback
|
||||
for bad_scale in (s.new_empty(s.shape), s[:, :-1], s.cpu()):
|
||||
with self.assertRaisesRegex(ValueError, "packed UE8M0"):
|
||||
linear(
|
||||
q,
|
||||
layer.weight,
|
||||
[128, 128],
|
||||
layer.weight_scale_inv,
|
||||
bad_scale,
|
||||
)
|
||||
# This shape normally takes Triton, which cannot read packed A scales.
|
||||
with self.assertRaisesRegex(ValueError, "packed UE8M0"):
|
||||
linear(
|
||||
q, layer.weight[:65], [128, 128], layer.weight_scale_inv, s
|
||||
)
|
||||
|
||||
@unittest.skipUnless(
|
||||
get_device_sm() == 90, "SM90 UE8M0 activation-scale regression"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user