[Perf] [AMD] dsv4: optional e8m0 mxscale GEMM for dense 128x128 fp8 linears (6/8) (#44007)

Co-authored-by: Duyi-Wang <duyi.wang@amd.com>
Co-authored-by: HAI <hixiao@gmail.com>
This commit is contained in:
kk
2026-10-16 02:28:31 -07:00
committed by GitHub
co-authored by Duyi-Wang HAI
parent 8d229d96e0
commit 42eb556e66
7 changed files with 602 additions and 3 deletions
@@ -0,0 +1,98 @@
"""Fused RMSNorm + per-token 1x128 fp8 quant with power-of-two (ue8m0) scales.
The e8m0 counterpart of aiter's ``fused_rms_fp8_group_quant(...,
output_unquantized_inp1=True, transpose_scale=True)``: one launch emits the bf16
normed row (for the readers that want it unquantized) and the fp8 operand of an
e8m0 mxscale GEMM. The scale is ``ceil_pow2(amax / 448)`` (aiter's default
RoundUp / DeepSeek-V4 reference ``act_quant(scale_fmt="ue8m0")``), stored as one
e8m0 byte per group in column-major order: a contiguous ``[K/128, M]`` buffer
viewed as ``[M, K/128]``, the layout ``per_group_quant_hip(transpose_scale=True,
scale_type=fp8_e8m0)`` emits and aiter's mxscale bpreshuffle GEMM reads.
"""
from typing import Tuple
import torch
import triton
import triton.language as tl
_FP8_MAX = 448.0
@triton.jit
def _rmsnorm_fp8_e8m0_kernel(
x_ptr,
w_ptr,
y_ptr,
q_ptr,
s_ptr,
M,
N,
x_stride,
y_stride,
q_stride,
eps,
BLOCK_N: tl.constexpr,
GROUP: tl.constexpr,
NUM_G: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
offs = tl.arange(0, BLOCK_N)
mask = offs < N
x = tl.load(x_ptr + row * x_stride + offs, mask=mask, other=0.0).to(tl.float32)
rstd = tl.math.rsqrt(tl.sum(x * x, axis=0) / N + eps)
w = tl.load(w_ptr + offs, mask=mask, other=0.0).to(tl.float32)
y = (x * rstd * w).to(y_ptr.dtype.element_ty)
tl.store(y_ptr + row * y_stride + offs, y, mask=mask)
# Quantize the rounded bf16 row, as a separate quant of the norm output would.
yg = tl.reshape(y.to(tl.float32), (NUM_G, GROUP))
amax = tl.max(tl.abs(yg), axis=1)
bits = (amax * (1.0 / 448.0)).to(tl.int32, bitcast=True)
e = (bits >> 23) & 0xFF
e = tl.where(((bits & 0x7FFFFF) != 0) & (e < 0xFF), e + 1, e)
inv = tl.where(e > 0, 1.0 / (e << 23).to(tl.float32, bitcast=True), 0.0)
q = tl.clamp(yg * inv[:, None], -448.0, 448.0).to(q_ptr.dtype.element_ty)
q_offs = tl.reshape(offs, (NUM_G, GROUP))
tl.store(
q_ptr + row * q_stride + q_offs,
q,
mask=tl.reshape(mask, (NUM_G, GROUP)),
)
g = tl.arange(0, NUM_G)
tl.store(s_ptr + g * M + row, e.to(tl.uint8), mask=g * GROUP < N)
def rmsnorm_fp8_e8m0_quant(
x: torch.Tensor, weight: torch.Tensor, eps: float, group_size: int = 128
) -> Tuple[Tuple[torch.Tensor, torch.Tensor], torch.Tensor]:
"""Returns ``((q_fp8 [M, N], scale_e8m0 [M, N/128] column-major bytes), y [M, N])``."""
assert x.dim() == 2 and x.stride(-1) == 1
m, n = x.shape
assert n % group_size == 0
num_g = n // group_size
y = torch.empty((m, n), dtype=x.dtype, device=x.device)
q = torch.empty((m, n), dtype=torch.float8_e4m3fn, device=x.device)
s = torch.empty((num_g, m), dtype=torch.float8_e8m0fnu, device=x.device)
if m > 0:
num_g_pow2 = triton.next_power_of_2(num_g)
block_n = num_g_pow2 * group_size
num_warps = 4 if block_n <= 2048 else (8 if block_n <= 4096 else 16)
_rmsnorm_fp8_e8m0_kernel[(m,)](
x,
weight,
y,
q,
s.view(torch.uint8),
m,
n,
x.stride(0),
y.stride(0),
q.stride(0),
eps,
BLOCK_N=block_n,
GROUP=group_size,
NUM_G=num_g_pow2,
num_warps=num_warps,
)
return (q, s.view(m, num_g)), y
+9
View File
@@ -1613,6 +1613,15 @@ class Envs:
# only way to ask; on separate-KV it is the reverse -- --kv-cache-dtype
# picks the buffer dtype and this switch is inert.
SGLANG_DSV4_UNIFIED_KV_FP8 = EnvBool(False)
# ROCm gfx950, DeepSeek-V4 128x128 block-fp8 dense linears: run the GEMM as
# aiter's e8m0 mxscale bpreshuffle (flydsl) instead of the fp32-scale
# CK/asm/triton bpreshuffle. The weight bytes are shared; only an e8m0 copy
# of the (power-of-two) weight scale is added at load, and activations are
# quantized with power-of-two (ue8m0) scales. Restricted to the (N, K)
# shapes in SGLANG_DSV4_DENSE_MXSCALE_SHAPES ("N:K,N:K"; empty = built-in
# whitelist of shapes that win in the microbench, "all" = every shape).
SGLANG_DSV4_DENSE_MXSCALE = EnvBool(False)
SGLANG_DSV4_DENSE_MXSCALE_SHAPES = EnvStr("")
# fp8 unified_kv decode/verify on aiter's FlyDSL v4 nm decode when the
# installed aiter ships it; 0 keeps the asm kernel.
SGLANG_DSV4_FLYDSL_DECODE = EnvBool(True)
@@ -56,11 +56,13 @@ from sglang.srt.layers.quantization.fp8_utils import (
_deepgemm_block_fp8_supported,
_use_aiter_bpreshuffle_gfx95,
apply_fp8_linear,
block_fp8_scale_to_e8m0_128,
block_fp8_scale_to_mxfp8_e8m0,
can_auto_enable_marlin_fp8,
can_serve_block_fp8_as_mxfp8,
cutlass_fp8_supported,
deepgemm_w8a8_block_fp8_linear_with_fallback,
dense_mxscale_wanted,
dispatch_block_fp8_mxfp8_linear,
dispatch_w8a8_block_fp8_linear,
dispatch_w8a8_mxfp8_linear,
@@ -183,6 +185,7 @@ if _use_aiter:
ACTIVATION_SCHEMES = ["static", "dynamic"]
logger = logging.getLogger(__name__)
_DENSE_MXSCALE_LOGGED: set = set()
DSV4_DEQUANT_FP4_TABLE = torch.tensor(
[
@@ -902,6 +905,7 @@ class Fp8LinearMethod(LinearMethodBase):
# instead of silently reading a permuted weight.
layer.aiter_bpreshuffled = True
layer.weight.is_shuffled = True
self._maybe_prepare_dense_mxscale(layer)
if (
is_xpu()
@@ -926,6 +930,36 @@ class Fp8LinearMethod(LinearMethodBase):
with torch.no_grad():
layer.weight_scale_inv.set_(scale_reordered)
def _maybe_prepare_dense_mxscale(self, layer: Module) -> None:
"""SGLANG_DSV4_DENSE_MXSCALE: give a (16, 16)-preshuffled 128x128 block-fp8
weight an e8m0 copy of its scale, which routes aiter_w8a8_block_fp8_linear
onto aiter's mxscale bpreshuffle GEMM. Lossless only for power-of-two
scales (ue8m0 checkpoints); any other layer stays on the fp32 GEMM. The
weight bytes are shared, the copy costs N*K/16384 bytes."""
if self.weight_block_size != [128, 128] or self.convert_mxfp8_to_block:
return
if layer.weight.dtype != torch.float8_e4m3fn:
return
n, k = layer.weight.shape
if not dense_mxscale_wanted(n, k):
return
scale = layer.weight_scale_inv.data
if tuple(scale.shape) != (n // 128, k // 128) or n % 128 or k % 128:
return
e8m0 = block_fp8_scale_to_e8m0_128(scale)
if e8m0 is None:
logger.warning(
"Dense mxscale: %s keeps the fp32 GEMM (scales are not powers of two)",
getattr(layer, "prefix", tuple(layer.weight.shape)),
)
return
layer.weight._mxscale_w_scale = e8m0
if (n, k) not in _DENSE_MXSCALE_LOGGED:
_DENSE_MXSCALE_LOGGED.add((n, k))
logger.info(
"Dense mxscale (e8m0 bpreshuffle GEMM) enabled for N=%d K=%d", n, k
)
def _prepare_block_fp8_as_mxfp8(self, layer: Module) -> None:
layer.block_fp8_mxfp8_ready = False
if getattr(layer, "keep_plain_weight_layout", False):
@@ -121,6 +121,121 @@ def set_force_ck_w8a8(enabled: bool = True) -> None:
_FORCE_CK_W8A8 = enabled
# DeepSeek-V4 dense 128x128 block-fp8 linears on aiter's e8m0 mxscale bpreshuffle
# GEMM (SGLANG_DSV4_DENSE_MXSCALE). DeepseekV4 enables it via
# set_dense_mxscale(True) before its weights load, like set_force_ck_w8a8.
_DENSE_MXSCALE: bool = False
# (N, K) where the e8m0 GEMM (with aiter's dsv4 mxscale tuned rows) beats the
# fp32-scale bpreshuffle GEMM (incl. aiter #6004/#6029 tuned rows) on gfx950,
# cold-weight, over the decode M range. See dsv4_perf_opt reports/dense_gemm.md.
# (512, 7168) (DSpark draft wkv) is untuned on the e8m0 side and loses at M>=96.
DSV4_DENSE_MXSCALE_DEFAULT_SHAPES = frozenset(
{
(2048, 7168), # wqkv_a
(1536, 7168), # wq_a (unfused: DSpark draft)
(8192, 1536), # wq_b TP8, indexer wq_b
(16384, 1536), # wq_b TP4
(65536, 1536), # wq_b attn-TP1 (DP attention)
(7168, 2048), # wo_b TP8
(7168, 4096), # wo_b TP4
(7168, 16384), # wo_b attn-TP1 (DP attention)
(6144, 7168), # shared expert w13, TP1
(7168, 3072), # shared expert w2, TP1
(768, 7168), # shared expert w13, TP8
(7168, 384), # shared expert w2, TP8
(7168, 21504), # DSpark main_proj
}
)
@lru_cache(maxsize=1)
def aiter_has_dense_mxscale() -> bool:
"""Whether the installed aiter dispatches e8m0 block scales in
gemm_a8w8_blockscale_bpreshuffle. Older aiter takes the e8m0 bytes as fp32
scales and silently returns wrong results."""
try:
from aiter.ops.gemm_op_a8w8 import get_mxscale_bpreshuffle_config # noqa: F401
except Exception:
return False
return True
def set_dense_mxscale(enabled: bool = True) -> None:
global _DENSE_MXSCALE
if enabled and not aiter_has_dense_mxscale():
logger.warning(
"SGLANG_DSV4_DENSE_MXSCALE is set but the installed aiter has no e8m0 "
"mxscale bpreshuffle GEMM; keeping the fp32-scale GEMM."
)
enabled = False
_DENSE_MXSCALE = enabled
@lru_cache(maxsize=1)
def _dense_mxscale_shapes() -> Optional[frozenset]:
"""The (N, K) whitelist, or None for every shape."""
spec = envs.SGLANG_DSV4_DENSE_MXSCALE_SHAPES.get().strip()
if not spec:
return DSV4_DENSE_MXSCALE_DEFAULT_SHAPES
if spec.lower() == "all":
return None
shapes = set()
for item in spec.split(","):
n, k = item.strip().split(":")
shapes.add((int(n), int(k)))
return frozenset(shapes)
def dense_mxscale_wanted(n: int, k: int) -> bool:
if not (_DENSE_MXSCALE and _use_aiter_bpreshuffle_gfx95):
return False
shapes = _dense_mxscale_shapes()
return shapes is None or (n, k) in shapes
def block_fp8_scale_to_e8m0_128(scale: torch.Tensor) -> Optional[torch.Tensor]:
"""fp32 128x128 block scales [N/128, K/128] -> e8m0 bytes of the same shape,
or None if any scale is not a positive normal power of two (not lossless)."""
s = scale.detach().float().contiguous()
bits = s.view(torch.int32)
exp = (bits >> 23) & 0xFF
ok = (s > 0) & ((bits & 0x7FFFFF) == 0) & (exp > 0) & (exp < 0xFF)
if not bool(ok.all()):
return None
return exp.to(torch.uint8).view(torch.float8_e8m0fnu).contiguous()
def is_e8m0_scale(scale: Optional[torch.Tensor]) -> bool:
return scale is not None and scale.dtype == torch.float8_e8m0fnu
def dense_mxscale_ready(*linears) -> bool:
"""Whether every given linear consumes e8m0 (mxscale) activations, i.e. a
shared producer of their fp8 operand should emit ue8m0 scales."""
if not _DENSE_MXSCALE:
return False
for linear in linears:
if linear is None:
continue
weight = getattr(linear, "weight", None)
if getattr(weight, "_mxscale_w_scale", None) is None:
return False
return True
def quant_fp8_e8m0_128(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
"""Per-token 1x128 fp8 quant with ue8m0 scales, the scale bytes column-major
([K/128, M] storage viewed as [M, K/128]) as the mxscale GEMM reads them."""
return per_group_quant_hip(
x,
quant_dtype=aiter.dtypes.fp8,
group_size=128,
transpose_scale=True,
scale_type=torch.float8_e8m0fnu,
)
def materialize_bpreshuffle_fp8_scale(scale: torch.Tensor) -> torch.Tensor:
"""Materialize the physical scale layout consumed by gfx95 bpreshuffle GEMM."""
return scale.t().contiguous().t() if scale.dim() == 2 else scale
@@ -235,6 +350,8 @@ if _use_aiter:
)
aiter_per1x128_quant = get_hip_quant(aiter.QuantType.per_1x128)
from aiter.ops.quant import per_group_quant_hip
from sglang.kernels.ops.gemm.smallm_fp8_gfx950 import (
smallm_fp8_gemm,
smallm_fp8_gemm_supported,
@@ -1474,6 +1591,30 @@ def aiter_w8a8_block_fp8_linear(
)
return out.to(input.dtype).view(*output_shape)
# DeepSeek-V4 e8m0 mxscale route (SGLANG_DSV4_DENSE_MXSCALE): same
# (16, 16)-preshuffled weight, e8m0 copy of its scale. A producer that
# already emitted an fp32-scale operand keeps the fp32 GEMM below.
mx_w_scale = getattr(weight, "_mxscale_w_scale", None)
if is_e8m0_scale(input_scale) or (mx_w_scale is not None and input_scale is None):
if mx_w_scale is None:
raise RuntimeError(
"e8m0-scaled fp8 activation reached a linear without an e8m0 "
f"weight scale (weight {tuple(weight.shape)})"
)
out_dtype = torch.bfloat16 if input_scale is not None else input_2d.dtype
if input_2d.shape[0] == 0:
return input_2d.new_empty(output_shape, dtype=out_dtype)
if input_scale is None:
q_input, x_scale = quant_fp8_e8m0_128(input_2d)
else:
q_input, x_scale = input_2d, input_scale
output = gemm_a8w8_blockscale_bpreshuffle(
q_input, weight, x_scale, mx_w_scale, dtype=torch.bfloat16
)
if bias is not None:
output += bias
return output.to(dtype=out_dtype).view(*output_shape)
n, k = weight.shape
if _use_aiter_bpreshuffle_gfx95:
@@ -180,6 +180,26 @@ def hc_post(cfg, x, residual, post, comb) -> Optional[torch.Tensor]:
return mhc_post_split_h(x, residual, post, comb, block_size=2048)
def _x_quant_e8m0(layer) -> bool:
"""Whether input_layernorm's fp8 operand goes out with ue8m0 scales: only when
every dense projection reading it (wqkv_a, or wq_a + wkv) runs the e8m0 mxscale
GEMM (SGLANG_DSV4_DENSE_MXSCALE). Resolved on first use, after weight load."""
cached = layer.__dict__.get("_x_quant_e8m0")
if cached is None:
from sglang.srt.layers.quantization.fp8_utils import dense_mxscale_ready
attn = layer.self_attn
if getattr(attn, "fuse_wqa_wkv", False):
consumers = (getattr(attn, "wqkv_a", None),)
else:
consumers = (getattr(attn, "wq_a", None), getattr(attn, "wkv", None))
cached = all(c is not None for c in consumers) and dense_mxscale_ready(
*consumers
)
layer.__dict__["_x_quant_e8m0"] = cached
return cached
def input_norm(
layer,
hidden_states: torch.Tensor,
@@ -194,7 +214,10 @@ def input_norm(
if coefficients is not None:
coefficients.materialize()
x_quant, hidden_states = fused_rmsnorm_fp8_quant(
hidden_states, layer.input_layernorm.weight, layer.rms_norm_eps
hidden_states,
layer.input_layernorm.weight,
layer.rms_norm_eps,
e8m0=_x_quant_e8m0(layer),
)
return hidden_states, x_quant
if layer.fused_rmsnorm_fake_quant:
+32 -2
View File
@@ -551,7 +551,14 @@ def _apply_wo_a_bf16_matmul(
return torch.einsum("tgd,grd->tgr", o, wo_a)
def _fused_rmsnorm_fp8_quant(hidden_states, weight, eps):
def _fused_rmsnorm_fp8_quant(hidden_states, weight, eps, e8m0: bool = False):
if e8m0:
# Consumers run the e8m0 mxscale GEMM (SGLANG_DSV4_DENSE_MXSCALE).
from sglang.kernels.ops.layernorm.rmsnorm_fp8_e8m0 import (
rmsnorm_fp8_e8m0_quant,
)
return rmsnorm_fp8_e8m0_quant(hidden_states, weight, eps)
x_quant, x_bf16, _, _ = fused_rms_fp8_group_quant(
hidden_states,
weight,
@@ -1126,6 +1133,22 @@ class MqaAttentionBase(nn.Module):
self._attn_sink_local = sink
return self._attn_sink_local[:kernel_num_heads]
def _q_lora_quant_e8m0(self, with_indexer: bool) -> bool:
"""Whether the fused q_norm quant feeds wq_b (and the indexer's wq_b, which
reuses it) with ue8m0 scales: only when each consumer runs the e8m0 mxscale
GEMM. Resolved on first use, after the weights are processed."""
key = "_q_lora_e8m0_idx" if with_indexer else "_q_lora_e8m0"
cached = self.__dict__.get(key)
if cached is None:
from sglang.srt.layers.quantization.fp8_utils import dense_mxscale_ready
indexer = getattr(self, "indexer", None) if with_indexer else None
cached = dense_mxscale_ready(
self.wq_b, getattr(indexer, "wq_b", None) if indexer else None
)
self.__dict__[key] = cached
return cached
@contextmanager
def maybe_use_decode_attn_tp(self, forward_batch: ForwardBatch):
ctx = get_cp_decode_attn_tp_ctx()
@@ -1897,6 +1920,7 @@ class MQALayer(MqaAttentionBase):
q_lora,
self.q_norm.weight,
self.q_norm.variance_epsilon,
e8m0=self._q_lora_quant_e8m0(with_indexer=False),
)
q, _ = self.wq_b(q_for_wqb)
else:
@@ -2036,6 +2060,7 @@ class MQALayer(MqaAttentionBase):
q_lora,
self.q_norm.weight,
self.q_norm.variance_epsilon,
e8m0=self._q_lora_quant_e8m0(with_indexer=True),
)
q, _ = self.wq_b(q_for_wqb)
else:
@@ -4251,10 +4276,15 @@ class DeepseekV4ForCausalLM(nn.Module):
# Module-level toggles default OFF; flipped True here for DSV4
if _is_hip:
from sglang.kernels.ops.attention.deepseek_v4_rope import set_batched_rope
from sglang.srt.layers.quantization.fp8_utils import set_force_ck_w8a8
from sglang.srt.layers.quantization.fp8_utils import (
set_dense_mxscale,
set_force_ck_w8a8,
)
set_force_ck_w8a8(True)
set_batched_rope(True)
if envs.SGLANG_DSV4_DENSE_MXSCALE.get():
set_dense_mxscale(True)
self.config = config
self.tp_size = get_parallel().tp_size
self.quant_config = quant_config