From ed89693c4ac3f187792494723a240345eac440cf Mon Sep 17 00:00:00 2001 From: Khoa Pham Date: Wed, 21 Oct 2026 16:56:45 -0700 Subject: [PATCH] [Perf] dsv41: Fuse the q_norm MXFP8 quantize for wq_b at every row count (#45633) --- .../kernels/ops/layernorm/rmsnorm_mxfp8.py | 17 ++++++++++------- .../kernels/ops/quantization/mxfp8_epilogue.py | 5 +++-- python/sglang/srt/models/deepseek_v4.py | 2 +- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/python/sglang/kernels/ops/layernorm/rmsnorm_mxfp8.py b/python/sglang/kernels/ops/layernorm/rmsnorm_mxfp8.py index 3b076df318..5c8ffc856d 100644 --- a/python/sglang/kernels/ops/layernorm/rmsnorm_mxfp8.py +++ b/python/sglang/kernels/ops/layernorm/rmsnorm_mxfp8.py @@ -17,7 +17,7 @@ def _rmsnorm_mxfp8_kernel( Q, S, SX: tl.constexpr, - M: tl.constexpr, + M, K: tl.constexpr, EPS: tl.constexpr, BLOCK: tl.constexpr, @@ -34,9 +34,10 @@ def _rmsnorm_mxfp8_kernel( mxfp8_epilogue(y, row, Q, S, K, BLOCK, GROUPS, 0, GROUPS) else: # Padding scale entries are disjoint from the live rows above. - off = (row - M) * 512 + tl.arange(0, 512) - pad_row = ((off // 4) % 4) * 32 + ((off // 16) % 32) - tl.store(S + off, 0, (off < GROUPS * 128) & (pad_row >= M)) + last = (tl.cdiv(M, 128) - 1) * 128 + local = (row - M) * 512 + tl.arange(0, 512) + pad_row = last + ((local // 4) % 4) * 32 + ((local // 16) % 32) + tl.store(S + last * GROUPS + local, 0, (local < GROUPS * 128) & (pad_row >= M)) def rmsnorm_mxfp8( @@ -44,12 +45,14 @@ def rmsnorm_mxfp8( ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """RMSNorm returning ``(y_bf16, y_q, y_sf)`` with the same MXFP8 epilogue.""" m, k = x.shape - assert 0 < m <= 8 and k % 32 == 0 + assert m > 0 and k % 128 == 0 assert x.dtype == weight.dtype == torch.bfloat16 and x.stride(1) == 1 y = torch.empty_like(x, memory_format=torch.contiguous_format) q = torch.empty((m, k), dtype=torch.float8_e4m3fn, device=x.device) - s = torch.empty((k // 32) * 128, dtype=torch.uint8, device=x.device) - _rmsnorm_mxfp8_kernel[(m + triton.cdiv(s.numel(), 512),)]( + s = torch.empty( + (k // 32) * triton.cdiv(m, 128) * 128, dtype=torch.uint8, device=x.device + ) + _rmsnorm_mxfp8_kernel[(m + triton.cdiv((k // 32) * 128, 512),)]( x, weight, y, diff --git a/python/sglang/kernels/ops/quantization/mxfp8_epilogue.py b/python/sglang/kernels/ops/quantization/mxfp8_epilogue.py index f7c5a89b32..9ff4152bbf 100644 --- a/python/sglang/kernels/ops/quantization/mxfp8_epilogue.py +++ b/python/sglang/kernels/ops/quantization/mxfp8_epilogue.py @@ -8,7 +8,7 @@ rounding the standalone pair produces, so both outputs are bitwise identical to ``rmsnorm`` followed by ``mxfp8_quantize(..., is_sf_swizzled_layout=True)``. The scale factors use FlashInfer's 128x4 swizzle: -``off = (g // 4) * 512 + ((r % 32) * 4 + ((r // 32) % 4)) * 4 + (g % 4)`` +``off = (r // 128) * ceil(G / 4) * 512 + (g // 4) * 512 + ((r % 32) * 4 + ((r // 32) % 4)) * 4 + (g % 4)`` over a row count padded to a multiple of 128, with the UE8M0 conversion (positive rounding, subnormals included) that ``mxfp8_quantize`` uses. """ @@ -48,5 +48,6 @@ def mxfp8_epilogue( sf, scale = ue8m0_scale(amax) q = tl.minimum(tl.maximum(v * scale[:, None], -448.0), 448.0).to(tl.float8e4nv) tl.store(Q + row * K + idx, q, gmask[:, None]) - off = (g // 4) * 512 + ((row % 32) * 4 + ((row // 32) % 4)) * 4 + (g % 4) + tile = (row // 128) * (((GROUPS + 3) // 4) * 512) + off = tile + (g // 4) * 512 + ((row % 32) * 4 + ((row // 32) % 4)) * 4 + (g % 4) tl.store(S + off, sf.to(tl.uint8), gmask) diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index b3f709fa3d..da3489eb13 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -1581,7 +1581,7 @@ class MQALayer(MqaAttentionBase): and get_platform().is_blackwell and q.dtype == self.q_norm.weight.dtype == torch.bfloat16 and q.ndim == 2 - and 0 < q.shape[0] <= 8 + and q.shape[0] > 0 and q.shape[1] == 1280 and q.stride(1) == 1 and getattr(method, "mxfp8_dense_backend", None)