[Perf] dsv41: Fuse the q_norm MXFP8 quantize for wq_b at every row count (#45633)
This commit is contained in:
@@ -17,7 +17,7 @@ def _rmsnorm_mxfp8_kernel(
|
||||
Q,
|
||||
S,
|
||||
SX: tl.constexpr,
|
||||
M: tl.constexpr,
|
||||
M,
|
||||
K: tl.constexpr,
|
||||
EPS: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
@@ -34,9 +34,10 @@ def _rmsnorm_mxfp8_kernel(
|
||||
mxfp8_epilogue(y, row, Q, S, K, BLOCK, GROUPS, 0, GROUPS)
|
||||
else:
|
||||
# Padding scale entries are disjoint from the live rows above.
|
||||
off = (row - M) * 512 + tl.arange(0, 512)
|
||||
pad_row = ((off // 4) % 4) * 32 + ((off // 16) % 32)
|
||||
tl.store(S + off, 0, (off < GROUPS * 128) & (pad_row >= M))
|
||||
last = (tl.cdiv(M, 128) - 1) * 128
|
||||
local = (row - M) * 512 + tl.arange(0, 512)
|
||||
pad_row = last + ((local // 4) % 4) * 32 + ((local // 16) % 32)
|
||||
tl.store(S + last * GROUPS + local, 0, (local < GROUPS * 128) & (pad_row >= M))
|
||||
|
||||
|
||||
def rmsnorm_mxfp8(
|
||||
@@ -44,12 +45,14 @@ def rmsnorm_mxfp8(
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""RMSNorm returning ``(y_bf16, y_q, y_sf)`` with the same MXFP8 epilogue."""
|
||||
m, k = x.shape
|
||||
assert 0 < m <= 8 and k % 32 == 0
|
||||
assert m > 0 and k % 128 == 0
|
||||
assert x.dtype == weight.dtype == torch.bfloat16 and x.stride(1) == 1
|
||||
y = torch.empty_like(x, memory_format=torch.contiguous_format)
|
||||
q = torch.empty((m, k), dtype=torch.float8_e4m3fn, device=x.device)
|
||||
s = torch.empty((k // 32) * 128, dtype=torch.uint8, device=x.device)
|
||||
_rmsnorm_mxfp8_kernel[(m + triton.cdiv(s.numel(), 512),)](
|
||||
s = torch.empty(
|
||||
(k // 32) * triton.cdiv(m, 128) * 128, dtype=torch.uint8, device=x.device
|
||||
)
|
||||
_rmsnorm_mxfp8_kernel[(m + triton.cdiv((k // 32) * 128, 512),)](
|
||||
x,
|
||||
weight,
|
||||
y,
|
||||
|
||||
@@ -8,7 +8,7 @@ rounding the standalone pair produces, so both outputs are bitwise identical to
|
||||
``rmsnorm`` followed by ``mxfp8_quantize(..., is_sf_swizzled_layout=True)``.
|
||||
|
||||
The scale factors use FlashInfer's 128x4 swizzle:
|
||||
``off = (g // 4) * 512 + ((r % 32) * 4 + ((r // 32) % 4)) * 4 + (g % 4)``
|
||||
``off = (r // 128) * ceil(G / 4) * 512 + (g // 4) * 512 + ((r % 32) * 4 + ((r // 32) % 4)) * 4 + (g % 4)``
|
||||
over a row count padded to a multiple of 128, with the UE8M0 conversion
|
||||
(positive rounding, subnormals included) that ``mxfp8_quantize`` uses.
|
||||
"""
|
||||
@@ -48,5 +48,6 @@ def mxfp8_epilogue(
|
||||
sf, scale = ue8m0_scale(amax)
|
||||
q = tl.minimum(tl.maximum(v * scale[:, None], -448.0), 448.0).to(tl.float8e4nv)
|
||||
tl.store(Q + row * K + idx, q, gmask[:, None])
|
||||
off = (g // 4) * 512 + ((row % 32) * 4 + ((row // 32) % 4)) * 4 + (g % 4)
|
||||
tile = (row // 128) * (((GROUPS + 3) // 4) * 512)
|
||||
off = tile + (g // 4) * 512 + ((row % 32) * 4 + ((row // 32) % 4)) * 4 + (g % 4)
|
||||
tl.store(S + off, sf.to(tl.uint8), gmask)
|
||||
|
||||
@@ -1581,7 +1581,7 @@ class MQALayer(MqaAttentionBase):
|
||||
and get_platform().is_blackwell
|
||||
and q.dtype == self.q_norm.weight.dtype == torch.bfloat16
|
||||
and q.ndim == 2
|
||||
and 0 < q.shape[0] <= 8
|
||||
and q.shape[0] > 0
|
||||
and q.shape[1] == 1280
|
||||
and q.stride(1) == 1
|
||||
and getattr(method, "mxfp8_dense_backend", None)
|
||||
|
||||
Reference in New Issue
Block a user