[SM90] Use native block32 DeepGEMM for DeepSeek-V4.1 (#42604)
Co-authored-by: BBuf <1182563586@qq.com>
This commit is contained in:
co-authored by
BBuf
parent
465253ef52
commit
2ff3e6dce1
@@ -77,7 +77,7 @@ dependencies = [
|
||||
"sentencepiece==0.2.1", # 0.2.2 rejects null pieces in InternVL tokenizers.
|
||||
"setproctitle",
|
||||
"sgl-deep-ep==0.1.2.post1",
|
||||
"sgl-deep-gemm==0.2.1.post1",
|
||||
"sgl-deep-gemm==0.2.2",
|
||||
"sglang-kernel==0.4.9",
|
||||
"smg-grpc-servicer>=0.9.0",
|
||||
"soundfile==0.13.1",
|
||||
|
||||
@@ -33,8 +33,11 @@ def deep_gemm_fp8_fp8_bf16_nt(
|
||||
B: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
C: torch.Tensor,
|
||||
block_n: int = 128,
|
||||
block_k: int = 128,
|
||||
) -> None:
|
||||
deep_gemm_wrapper.gemm_nt_f8f8bf16((A, As), (B, Bs), C)
|
||||
recipe = (1, block_n, block_k) if (block_n, block_k) != (128, 128) else None
|
||||
deep_gemm_wrapper.gemm_nt_f8f8bf16((A, As), (B, Bs), C, recipe=recipe)
|
||||
|
||||
|
||||
@register_custom_op(mutates_args=["C"])
|
||||
@@ -746,7 +749,7 @@ def w8a8_block_fp8_matmul_deepgemm(
|
||||
# Deepgemm only supports output tensor type as bfloat16
|
||||
assert C.dtype == torch.bfloat16 and deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM
|
||||
|
||||
deep_gemm_fp8_fp8_bf16_nt(A, As, B, Bs, C)
|
||||
deep_gemm_fp8_fp8_bf16_nt(A, As, B, Bs, C, block_size[0], block_size[1])
|
||||
|
||||
return C
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import os
|
||||
import time
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from enum import IntEnum, auto
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
@@ -115,6 +116,7 @@ class DeepGemmKernelType(IntEnum):
|
||||
GROUPED_GEMM_NT_BF16_MASKED = auto()
|
||||
GROUPED_GEMM_NT_BF16_CONTIG = auto()
|
||||
GEMM_NT_F8F8BF16 = auto()
|
||||
GEMM_NT_F8F8BF16_BLOCK32 = auto()
|
||||
GEMM_NT_BF16BF16F32 = auto()
|
||||
|
||||
|
||||
@@ -269,6 +271,9 @@ class _BaseWarmupExecutor:
|
||||
def create(kernel_type: DeepGemmKernelType, **kwargs):
|
||||
return {
|
||||
DeepGemmKernelType.GEMM_NT_F8F8BF16: _NormalWarmupExecutor,
|
||||
DeepGemmKernelType.GEMM_NT_F8F8BF16_BLOCK32: partial(
|
||||
_NormalWarmupExecutor, block_size=32
|
||||
),
|
||||
DeepGemmKernelType.GROUPED_GEMM_NT_F8F8BF16_CONTIG: _GroupedContWarmupExecutor,
|
||||
DeepGemmKernelType.GROUPED_GEMM_NT_F8F8BF16_MASKED: _GroupedMaskedWarmupExecutor,
|
||||
DeepGemmKernelType.GEMM_NT_BF16BF16F32: _BF16F32WarmupExecutor,
|
||||
@@ -282,8 +287,17 @@ class _BaseWarmupExecutor:
|
||||
) -> float:
|
||||
# FP32 input scales remain live while DeepGEMM converts their layout.
|
||||
_GB = 1 << 30
|
||||
lhs_scales = max_m * ceil_div(k, _BLOCK_SIZE) * 4
|
||||
rhs_scales = ceil_div(n, _BLOCK_SIZE) * ceil_div(k, _BLOCK_SIZE) * 4
|
||||
is_dense_fp8 = kernel_type in (
|
||||
DeepGemmKernelType.GEMM_NT_F8F8BF16,
|
||||
DeepGemmKernelType.GEMM_NT_F8F8BF16_BLOCK32,
|
||||
)
|
||||
block_size = (
|
||||
32
|
||||
if kernel_type == DeepGemmKernelType.GEMM_NT_F8F8BF16_BLOCK32
|
||||
else _BLOCK_SIZE
|
||||
)
|
||||
lhs_scales = max_m * ceil_div(k, block_size) * 4
|
||||
rhs_scales = ceil_div(n, block_size) * ceil_div(k, block_size) * 4
|
||||
scale_workspace = _get_fp8_scale_workspace(
|
||||
max_m,
|
||||
n,
|
||||
@@ -293,11 +307,10 @@ class _BaseWarmupExecutor:
|
||||
if kernel_type == DeepGemmKernelType.GROUPED_GEMM_NT_F8F8BF16_MASKED
|
||||
else 1
|
||||
),
|
||||
rhs_groups=(
|
||||
1 if kernel_type == DeepGemmKernelType.GEMM_NT_F8F8BF16 else num_groups
|
||||
),
|
||||
rhs_groups=1 if is_dense_fp8 else num_groups,
|
||||
block_size=block_size,
|
||||
)
|
||||
if kernel_type == DeepGemmKernelType.GEMM_NT_F8F8BF16:
|
||||
if is_dense_fp8:
|
||||
return (
|
||||
max_m * k
|
||||
+ n * k
|
||||
@@ -358,9 +371,9 @@ def _packed_scale_workspace(rows: int, cols: int, groups: int) -> tuple[int, int
|
||||
|
||||
|
||||
def _get_fp8_scale_workspace(
|
||||
max_m: int, n: int, k: int, lhs_groups: int, rhs_groups: int
|
||||
max_m: int, n: int, k: int, lhs_groups: int, rhs_groups: int, block_size: int = 128
|
||||
) -> int:
|
||||
scale_k = ceil_div(k, _BLOCK_SIZE)
|
||||
scale_k = ceil_div(k, block_size)
|
||||
if DEEPGEMM_NEED_TMA_ALIGNED_SCALES:
|
||||
return lhs_groups * ceil_align(max_m, 4) * scale_k * 4
|
||||
if DEEPGEMM_SCALE_UE8M0:
|
||||
@@ -373,22 +386,22 @@ def _get_fp8_scale_workspace(
|
||||
return 0
|
||||
|
||||
|
||||
def _empty_token_fp8(size):
|
||||
def _empty_token_fp8(size, block_size=128):
|
||||
*dims, k = size
|
||||
return (
|
||||
torch.empty(size, device="cuda", dtype=torch.float8_e4m3fn),
|
||||
torch.ones(
|
||||
(*dims, ceil_div(k, _BLOCK_SIZE)), device="cuda", dtype=torch.float32
|
||||
(*dims, ceil_div(k, block_size)), device="cuda", dtype=torch.float32
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _empty_block_fp8(size):
|
||||
def _empty_block_fp8(size, block_size=128):
|
||||
*dims, n, k = size
|
||||
return (
|
||||
torch.empty(size, device="cuda", dtype=torch.float8_e4m3fn),
|
||||
torch.ones(
|
||||
(*dims, ceil_div(n, _BLOCK_SIZE), ceil_div(k, _BLOCK_SIZE)),
|
||||
(*dims, ceil_div(n, block_size), ceil_div(k, block_size)),
|
||||
device="cuda",
|
||||
dtype=torch.float32,
|
||||
),
|
||||
@@ -399,9 +412,12 @@ _BLOCK_SIZE = 128
|
||||
|
||||
|
||||
class _NormalWarmupExecutor(_BaseWarmupExecutor):
|
||||
def __init__(self, max_m: int, n: int, k: int, num_groups: int):
|
||||
self.lhs_q, self.lhs_s = _empty_token_fp8((max_m, k))
|
||||
self.rhs_q, self.rhs_s = _empty_block_fp8((n, k))
|
||||
def __init__(
|
||||
self, max_m: int, n: int, k: int, num_groups: int, block_size: int = 128
|
||||
):
|
||||
self.recipe = (1, block_size, block_size)
|
||||
self.lhs_q, self.lhs_s = _empty_token_fp8((max_m, k), block_size)
|
||||
self.rhs_q, self.rhs_s = _empty_block_fp8((n, k), block_size)
|
||||
self.out = torch.empty((max_m, n), device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
def execute(self, m):
|
||||
@@ -409,6 +425,7 @@ class _NormalWarmupExecutor(_BaseWarmupExecutor):
|
||||
(self.lhs_q[:m], self.lhs_s[:m]),
|
||||
(self.rhs_q, self.rhs_s),
|
||||
self.out[:m],
|
||||
recipe=self.recipe,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -220,11 +220,16 @@ def gemm_nt_f8f8bf16(
|
||||
lhs: Tuple[torch.Tensor, torch.Tensor],
|
||||
rhs: Tuple[torch.Tensor, torch.Tensor],
|
||||
out: torch.Tensor,
|
||||
recipe: Optional[Tuple[int, int, int]] = None,
|
||||
):
|
||||
m, k = lhs[0].shape
|
||||
n, _ = rhs[0].shape
|
||||
num_groups = 1
|
||||
kernel_type = compile_utils.DeepGemmKernelType.GEMM_NT_F8F8BF16
|
||||
kernel_type = (
|
||||
compile_utils.DeepGemmKernelType.GEMM_NT_F8F8BF16_BLOCK32
|
||||
if recipe == (1, 32, 32)
|
||||
else compile_utils.DeepGemmKernelType.GEMM_NT_F8F8BF16
|
||||
)
|
||||
|
||||
_sanity_check_input(lhs)
|
||||
_sanity_check_input(rhs)
|
||||
@@ -234,6 +239,7 @@ def gemm_nt_f8f8bf16(
|
||||
lhs,
|
||||
rhs,
|
||||
out,
|
||||
**({"recipe": recipe} if recipe is not None else {}),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -53,6 +53,7 @@ from sglang.srt.layers.quantization.base_config import (
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from sglang.srt.layers.quantization.fp8_utils import (
|
||||
_deepgemm_block_fp8_supported,
|
||||
_use_aiter_bpreshuffle_gfx95,
|
||||
apply_fp8_linear,
|
||||
block_fp8_scale_to_mxfp8_e8m0,
|
||||
@@ -729,6 +730,10 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
)
|
||||
|
||||
def process_weights_after_loading_block_quant(self, layer: Module) -> None:
|
||||
use_deepgemm_runner = (
|
||||
getattr(self.w8a8_block_fp8_linear, "func", self.w8a8_block_fp8_linear)
|
||||
is deepgemm_w8a8_block_fp8_linear_with_fallback
|
||||
)
|
||||
if self.convert_mxfp8_to_block:
|
||||
from sglang.srt.layers.quantization.mxfp8_block_convert import (
|
||||
convert_mxfp8_weight_to_block_fp8,
|
||||
@@ -810,10 +815,6 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
return
|
||||
else:
|
||||
# Requantize block scales to UE8M0 when DeepGEMM is the active runner.
|
||||
use_deepgemm_runner = (
|
||||
self.w8a8_block_fp8_linear
|
||||
is deepgemm_w8a8_block_fp8_linear_with_fallback
|
||||
)
|
||||
requant_block_scale_ue8m0_for_deepgemm(
|
||||
layer.weight,
|
||||
layer.weight_scale_inv,
|
||||
@@ -833,6 +834,14 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
_is_cuda
|
||||
and get_platform().is_sm90
|
||||
and envs.SGLANG_OPT_HOPPER_BLOCK_FP8_BF16.get()
|
||||
and not (
|
||||
use_deepgemm_runner
|
||||
and _deepgemm_block_fp8_supported(
|
||||
weight.shape,
|
||||
self.weight_block_size,
|
||||
getattr(layer, "orig_dtype", None),
|
||||
)
|
||||
)
|
||||
and weight.is_cuda
|
||||
and weight.dtype == torch.float8_e4m3fn
|
||||
and self.weight_block_size == [32, 32]
|
||||
|
||||
@@ -620,19 +620,32 @@ def dispatch_w8a8_block_fp8_linear(
|
||||
1. The --fp8-gemm-backend server argument (preferred)
|
||||
2. Auto-detection based on hardware capabilities
|
||||
"""
|
||||
# Only Triton reads the block size at launch; DeepGEMM, the FlashInfer
|
||||
# groupwise kernels and CUTLASS take 128-wide K blocks only.
|
||||
backend = get_fp8_gemm_runner_backend()
|
||||
if (
|
||||
weight_block_size == [32, 32]
|
||||
and get_platform().is_sm90
|
||||
and deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM
|
||||
and (backend.is_auto() or backend.is_deep_gemm())
|
||||
):
|
||||
return partial(
|
||||
deepgemm_w8a8_block_fp8_linear_with_fallback,
|
||||
act_scale_ue8m0=act_scale_ue8m0,
|
||||
)
|
||||
|
||||
# Other non-128 layouts still require Triton or the separate MXFP8 route.
|
||||
if weight_block_size is not None and weight_block_size[1] != 128:
|
||||
return partial(triton_w8a8_block_fp8_linear, act_scale_ue8m0=act_scale_ue8m0)
|
||||
|
||||
backend = get_fp8_gemm_runner_backend()
|
||||
|
||||
# Handle explicit backend selection via --fp8-gemm-backend
|
||||
if not backend.is_auto():
|
||||
return _dispatch_explicit_backend(backend)
|
||||
|
||||
# Auto mode: Select based purely on hardware/backend availability
|
||||
return _dispatch_auto_backend()
|
||||
linear = _dispatch_explicit_backend(backend)
|
||||
else:
|
||||
# Auto mode: Select based purely on hardware/backend availability
|
||||
linear = _dispatch_auto_backend()
|
||||
# Preserve the unwrapped callable when no activation policy needs binding.
|
||||
if act_scale_ue8m0 and linear is deepgemm_w8a8_block_fp8_linear_with_fallback:
|
||||
return partial(linear, act_scale_ue8m0=True)
|
||||
return linear
|
||||
|
||||
|
||||
def torch_w8a8_block_fp8_linear(
|
||||
@@ -1268,6 +1281,17 @@ def cutlass_w8a8_block_fp8_linear_with_fallback(
|
||||
return output.to(dtype=input_2d.dtype).view(*output_shape)
|
||||
|
||||
|
||||
def _deepgemm_block_fp8_supported(
|
||||
weight_shape: Tuple[int, int], block_size: List[int], output_dtype: torch.dtype
|
||||
) -> bool:
|
||||
if output_dtype != torch.bfloat16:
|
||||
return False
|
||||
n, k = weight_shape
|
||||
if list(block_size) == [32, 32] and get_platform().is_sm90:
|
||||
return n % 8 == 0 and k % 32 == 0
|
||||
return list(block_size) == [128, 128] and n % 64 == 0 and k % 128 == 0
|
||||
|
||||
|
||||
def deepgemm_w8a8_block_fp8_linear_with_fallback(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
@@ -1275,43 +1299,16 @@ def deepgemm_w8a8_block_fp8_linear_with_fallback(
|
||||
weight_scale: torch.Tensor,
|
||||
input_scale: Optional[torch.Tensor] = None,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
act_scale_ue8m0: bool = False,
|
||||
weight_bf16: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
if input_scale is not None:
|
||||
# Pre-quantized activation (SGLANG_OPT_MOE_QUANT_ONCE): ``input`` is
|
||||
# the fp8 per-token-group-128 q with rows padded to a multiple of 4
|
||||
# and ``input_scale`` the matching column-major fp32 scales
|
||||
# (stride == (1, padded_rows)) -- identical to the MN-major
|
||||
# TMA-aligned layout this path's own quant would produce below.
|
||||
# Output keeps the padded row count; the caller slices back.
|
||||
# UE8M0 packed scales (Blackwell DeepGEMM) use a different layout;
|
||||
# the caller gates on it.
|
||||
assert not deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
|
||||
assert input.dtype == torch.float8_e4m3fn
|
||||
assert weight.shape[0] % 64 == 0 and weight.shape[1] % 128 == 0, (
|
||||
"pre-quantized fp8 input requires DeepGEMM-supported weight shapes "
|
||||
f"(got {tuple(weight.shape)})"
|
||||
)
|
||||
input_2d = input.view(-1, input.shape[-1])
|
||||
output = w8a8_block_fp8_matmul_deepgemm(
|
||||
input_2d,
|
||||
weight,
|
||||
input_scale,
|
||||
weight_scale,
|
||||
block_size,
|
||||
output_dtype=torch.bfloat16,
|
||||
)
|
||||
if bias is not None:
|
||||
output += bias
|
||||
return output.view(*input.shape[:-1], weight.shape[0])
|
||||
|
||||
output_dtype = input.dtype
|
||||
dtype_supported = output_dtype == torch.bfloat16
|
||||
output_dtype = torch.bfloat16 if input_scale is not None else input.dtype
|
||||
|
||||
# TODO: https://github.com/sgl-project/sglang/pull/6890#issuecomment-2943395737
|
||||
shape_supported = weight.shape[0] % 64 == 0 and weight.shape[1] % 128 == 0
|
||||
block_supported = list(block_size) == [128, 128]
|
||||
|
||||
if not (shape_supported and dtype_supported and block_supported):
|
||||
if not _deepgemm_block_fp8_supported(weight.shape, block_size, output_dtype):
|
||||
# fall back to triton
|
||||
# If weight_scale is in UE8M0 packed format (int32), convert back to float32
|
||||
# UE8M0 format has shape (N, K//block_k//4) with dtype int32
|
||||
@@ -1321,13 +1318,42 @@ def deepgemm_w8a8_block_fp8_linear_with_fallback(
|
||||
weight_scale, weight.shape, block_size
|
||||
)
|
||||
return triton_w8a8_block_fp8_linear(
|
||||
input, weight, block_size, weight_scale, input_scale, bias
|
||||
input,
|
||||
weight,
|
||||
block_size,
|
||||
weight_scale,
|
||||
input_scale,
|
||||
bias,
|
||||
act_scale_ue8m0=act_scale_ue8m0,
|
||||
weight_bf16=weight_bf16,
|
||||
)
|
||||
|
||||
if input_scale is not None:
|
||||
# Pre-quantized activations retain their per-token group size and
|
||||
# fp32 scales. DeepGEMM aligns scale storage for TMA when needed.
|
||||
input_2d = input.view(-1, input.shape[-1])
|
||||
output = w8a8_block_fp8_matmul_deepgemm(
|
||||
input_2d,
|
||||
weight,
|
||||
input_scale,
|
||||
weight_scale,
|
||||
block_size,
|
||||
output_dtype=output_dtype,
|
||||
)
|
||||
if bias is not None:
|
||||
output += bias
|
||||
return output.view(*input.shape[:-1], weight.shape[0])
|
||||
|
||||
input_2d = input.view(-1, input.shape[-1])
|
||||
output_shape = [*input.shape[:-1], weight.shape[0]]
|
||||
|
||||
if not _is_musa:
|
||||
if get_platform().is_sm90 and act_scale_ue8m0:
|
||||
# Keep the checkpoint's power-of-two activation quantization. SM90
|
||||
# consumes fp32 scales; DeepGEMM aligns their storage for TMA.
|
||||
q_input, x_scale = sglang_per_token_group_quant_fp8(
|
||||
input_2d, block_size[1], scale_ue8m0=True
|
||||
)
|
||||
elif not _is_musa:
|
||||
q_input, x_scale = sglang_per_token_group_quant_fp8(
|
||||
input_2d,
|
||||
block_size[1],
|
||||
|
||||
@@ -1775,7 +1775,8 @@ class DeepseekV2MoE(nn.Module):
|
||||
if gate_up.weight.shape[0] % 64 != 0 or gate_up.weight.shape[1] % 128 != 0:
|
||||
return False, "gate_up weight shape unsupported by deepgemm"
|
||||
else:
|
||||
return False, f"w8a8 linear backend {linear_fn.__name__} unsupported"
|
||||
backend_name = getattr(linear_fn, "__name__", type(linear_fn).__name__)
|
||||
return False, f"w8a8 linear backend {backend_name} unsupported"
|
||||
# Routed side: standard dispatcher + triton fused func with dynamic
|
||||
# per-token-group-128 fp8 activation quant.
|
||||
experts = self.experts
|
||||
|
||||
Reference in New Issue
Block a user