[Perf] dsv41: Fuse the q_norm MXFP8 quantize for wq_b at every row count (#45633)

This commit is contained in:
Khoa Pham
2026-10-21 16:56:45 -07:00
committed by GitHub
parent da7b97a737
commit ed89693c4a
3 changed files with 14 additions and 10 deletions
@@ -17,7 +17,7 @@ def _rmsnorm_mxfp8_kernel(
Q,
S,
SX: tl.constexpr,
M: tl.constexpr,
M,
K: tl.constexpr,
EPS: tl.constexpr,
BLOCK: tl.constexpr,
@@ -34,9 +34,10 @@ def _rmsnorm_mxfp8_kernel(
mxfp8_epilogue(y, row, Q, S, K, BLOCK, GROUPS, 0, GROUPS)
else:
# Padding scale entries are disjoint from the live rows above.
off = (row - M) * 512 + tl.arange(0, 512)
pad_row = ((off // 4) % 4) * 32 + ((off // 16) % 32)
tl.store(S + off, 0, (off < GROUPS * 128) & (pad_row >= M))
last = (tl.cdiv(M, 128) - 1) * 128
local = (row - M) * 512 + tl.arange(0, 512)
pad_row = last + ((local // 4) % 4) * 32 + ((local // 16) % 32)
tl.store(S + last * GROUPS + local, 0, (local < GROUPS * 128) & (pad_row >= M))
def rmsnorm_mxfp8(
@@ -44,12 +45,14 @@ def rmsnorm_mxfp8(
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""RMSNorm returning ``(y_bf16, y_q, y_sf)`` with the same MXFP8 epilogue."""
m, k = x.shape
assert 0 < m <= 8 and k % 32 == 0
assert m > 0 and k % 128 == 0
assert x.dtype == weight.dtype == torch.bfloat16 and x.stride(1) == 1
y = torch.empty_like(x, memory_format=torch.contiguous_format)
q = torch.empty((m, k), dtype=torch.float8_e4m3fn, device=x.device)
s = torch.empty((k // 32) * 128, dtype=torch.uint8, device=x.device)
_rmsnorm_mxfp8_kernel[(m + triton.cdiv(s.numel(), 512),)](
s = torch.empty(
(k // 32) * triton.cdiv(m, 128) * 128, dtype=torch.uint8, device=x.device
)
_rmsnorm_mxfp8_kernel[(m + triton.cdiv((k // 32) * 128, 512),)](
x,
weight,
y,
@@ -8,7 +8,7 @@ rounding the standalone pair produces, so both outputs are bitwise identical to
``rmsnorm`` followed by ``mxfp8_quantize(..., is_sf_swizzled_layout=True)``.
The scale factors use FlashInfer's 128x4 swizzle:
``off = (g // 4) * 512 + ((r % 32) * 4 + ((r // 32) % 4)) * 4 + (g % 4)``
``off = (r // 128) * ceil(G / 4) * 512 + (g // 4) * 512 + ((r % 32) * 4 + ((r // 32) % 4)) * 4 + (g % 4)``
over a row count padded to a multiple of 128, with the UE8M0 conversion
(positive rounding, subnormals included) that ``mxfp8_quantize`` uses.
"""
@@ -48,5 +48,6 @@ def mxfp8_epilogue(
sf, scale = ue8m0_scale(amax)
q = tl.minimum(tl.maximum(v * scale[:, None], -448.0), 448.0).to(tl.float8e4nv)
tl.store(Q + row * K + idx, q, gmask[:, None])
off = (g // 4) * 512 + ((row % 32) * 4 + ((row // 32) % 4)) * 4 + (g % 4)
tile = (row // 128) * (((GROUPS + 3) // 4) * 512)
off = tile + (g // 4) * 512 + ((row % 32) * 4 + ((row // 32) % 4)) * 4 + (g % 4)
tl.store(S + off, sf.to(tl.uint8), gmask)
+1 -1
View File
@@ -1581,7 +1581,7 @@ class MQALayer(MqaAttentionBase):
and get_platform().is_blackwell
and q.dtype == self.q_norm.weight.dtype == torch.bfloat16
and q.ndim == 2
and 0 < q.shape[0] <= 8
and q.shape[0] > 0
and q.shape[1] == 1280
and q.stride(1) == 1
and getattr(method, "mxfp8_dense_backend", None)