From 379ec90fb24afd050e0721ee7482c573e8a213cc Mon Sep 17 00:00:00 2001 From: DarkSharpness <76582120+DarkSharpness@users.noreply.github.com> Date: Sat, 3 Oct 2026 18:24:41 +0800 Subject: [PATCH] [DSv4.1] Fold q_rope_store into fused_q_norm_rope (#41657) Co-authored-by: Claude Opus 5 --- .../jit/csrc/deepseek_v4/main_norm_rope.cuh | 57 +++++--- .../kernels/ops/attention/dsv4/elementwise.py | 12 +- .../ops/attention/dsv4/q_rope_store.py | 126 ------------------ python/sglang/srt/models/deepseek_v4.py | 28 ++-- .../ops/attention/test_dsv4_q_rope_store.py | 18 +-- 5 files changed, 64 insertions(+), 177 deletions(-) delete mode 100644 python/sglang/kernels/ops/attention/dsv4/q_rope_store.py diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/main_norm_rope.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/main_norm_rope.cuh index 94ef8bc30f..1ccfb43448 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/main_norm_rope.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/main_norm_rope.cuh @@ -80,7 +80,7 @@ struct FusedQNormRopeParams { float eps; }; -template +template Q_KERNEL void fused_q_norm_rope(const __grid_constant__ FusedQNormRopeParams params) { using namespace device; @@ -129,24 +129,26 @@ Q_KERNEL void fused_q_norm_rope(const __grid_constant__ FusedQNormRopeParams par const auto freq = mem_freq.load(params.freqs_cis + position * kRopeDim); - float sum_of_squares = 0.0f; + if constexpr (kApplyNorm) { + float sum_of_squares = 0.0f; #pragma unroll - for (int i = 0; i < kLocalSize; ++i) { + for (int i = 0; i < kLocalSize; ++i) { #pragma unroll - for (int j = 0; j < kVecSize; ++j) { - const auto x = cast(input_vec[i][j]); - sum_of_squares += x * x; + for (int j = 0; j < kVecSize; ++j) { + const auto x = cast(input_vec[i][j]); + sum_of_squares += x * x; + } } - } - sum_of_squares = warp::reduce_sum(sum_of_squares); - const auto norm_factor = math::rsqrt(sum_of_squares / kHeadDim + params.eps); + sum_of_squares = warp::reduce_sum(sum_of_squares); + const auto norm_factor = math::rsqrt(sum_of_squares / kHeadDim + params.eps); #pragma unroll - for (int i = 0; i < kLocalSize; ++i) { + for (int i = 0; i < kLocalSize; ++i) { #pragma unroll - for (int j = 0; j < kVecSize; ++j) { - const auto x = cast(input_vec[i][j]); - input_vec[i][j] = cast(x * norm_factor); + for (int j = 0; j < kVecSize; ++j) { + const auto x = cast(input_vec[i][j]); + input_vec[i][j] = cast(x * norm_factor); + } } } @@ -172,17 +174,34 @@ Q_KERNEL void fused_q_norm_rope(const __grid_constant__ FusedQNormRopeParams par const auto elem = mem_elem.load(s_rope[warp_id]); const auto [x_real, x_imag] = cast(elem); const auto [freq_real, freq_imag] = freq; - const fp32x2_t rotated = { - x_real * freq_real - x_imag * freq_imag, - x_real * freq_imag + x_imag * freq_real, - }; +#ifdef USE_ROCM + // gfx950 batched RoPE rounds the sine product, then one fma with the cosine. Only the + // kApplyNorm = false specialization replaces a kernel that rounded that way, so only it + // matches it; the norm path keeps the order it has always had. + constexpr bool kBatchedRopeOrder = !kApplyNorm; +#else + constexpr bool kBatchedRopeOrder = false; +#endif + fp32x2_t rotated; + if constexpr (kBatchedRopeOrder) { + rotated = { + fmaf(x_real, freq_real, -__fmul_rn(x_imag, freq_imag)), + fmaf(x_imag, freq_real, __fmul_rn(x_real, freq_imag)), + }; + } else { + rotated = { + x_real * freq_real - x_imag * freq_imag, + x_real * freq_imag + x_imag * freq_real, + }; + } mem_elem.store(output_ptr + (kHeadDim - kRopeDim), cast(rotated)); } -template +// kApplyNorm = false only rotates the rope tail while copying Q into a (padded) output. +template struct FusedQNormRopeKernel { template - static constexpr auto kernel = fused_q_norm_rope; + static constexpr auto kernel = fused_q_norm_rope; static void forward( const tvm::ffi::TensorView q_input, diff --git a/python/sglang/kernels/ops/attention/dsv4/elementwise.py b/python/sglang/kernels/ops/attention/dsv4/elementwise.py index c0ab88af3e..a104b0bcc7 100644 --- a/python/sglang/kernels/ops/attention/dsv4/elementwise.py +++ b/python/sglang/kernels/ops/attention/dsv4/elementwise.py @@ -37,9 +37,10 @@ def _jit_main_q_norm_rope_module( dtype: torch.dtype, head_dim: int, rope_dim: int, + apply_norm: bool, ): """Main MLA path Q kernel: rmsnorm-self + RoPE, warp per (token, head).""" - args = make_cpp_args(dtype, head_dim, rope_dim, is_arch_support_pdl()) + args = make_cpp_args(dtype, head_dim, rope_dim, apply_norm, is_arch_support_pdl()) return load_jit( make_name("main_q_norm_rope"), *args, @@ -170,7 +171,7 @@ def fused_rope_inplace( def fused_q_norm_rope( q_input: torch.Tensor, q_output: torch.Tensor, - eps: float, + eps: Optional[float], freqs_cis: torch.Tensor, positions: torch.Tensor, ) -> None: @@ -178,10 +179,13 @@ def fused_q_norm_rope( head_dim = q_input.shape[-1] rope_dim = freqs_real.shape[-1] if _is_xpu: + assert eps is not None fused_q_norm_rope_xpu(q_input, q_output, freqs_real, positions, eps) else: - module = _jit_main_q_norm_rope_module(q_input.dtype, head_dim, rope_dim) - module.forward(q_input, q_output, freqs_real, positions, eps) + module = _jit_main_q_norm_rope_module( + q_input.dtype, head_dim, rope_dim, eps is not None + ) + module.forward(q_input, q_output, freqs_real, positions, eps or 0.0) def fused_q_indexer_rope_hadamard_quant( diff --git a/python/sglang/kernels/ops/attention/dsv4/q_rope_store.py b/python/sglang/kernels/ops/attention/dsv4/q_rope_store.py deleted file mode 100644 index e2073b9e59..0000000000 --- a/python/sglang/kernels/ops/attention/dsv4/q_rope_store.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Rotate Q while writing the padded buffer consumed by sparse attention.""" - -import torch -import triton -import triton.language as tl - -from sglang.kernels.jit.utils import is_arch_support_pdl -from sglang.srt.utils import is_hip - - -@triton.jit -def _q_rope_store( - X, - Y, - F, - POS, - SX: tl.constexpr, - SY: tl.constexpr, - IS_HIP: tl.constexpr, - USE_GDC: tl.constexpr = False, -): - row, head = tl.program_id(0), tl.program_id(1) - r = tl.arange(0, 512) - if USE_GDC: - tl.extra.cuda.gdc_wait() # X is the wq_b GEMM output; POS may be in-graph - value = tl.load(X + row * SX + head * 512 + r).to(tl.float32) - partner = tl.gather(value, r ^ 1, 0) - position = tl.load(POS + row) - cos = tl.load(F + position * 64 + (r - 448) // 2 * 2, r >= 448, 0) - sin = tl.load(F + position * 64 + (r - 448) // 2 * 2 + 1, r >= 448, 0) - # Match the operation order of deepseek_rope_kernel before BF16 rounding. - even = tl.fma(value, cos, -partner * sin) - if IS_HIP: - # Match AMD flat RoPE: round the sine product before the cosine FMA. - odd = tl.fma(value, cos, partner * sin) - else: - odd = tl.fma(partner, sin, value * cos) - rotated = tl.where((r & 1) == 0, even, odd) - tl.store(Y + row * SY + head * 512 + r, tl.where(r >= 448, rotated, value)) - if USE_GDC: - tl.extra.cuda.gdc_launch_dependents() - - -@triton.jit -def _q_rope_store_prefill( - X, - Y, - F, - POS, - M, - H: tl.constexpr, - SX: tl.constexpr, - SY: tl.constexpr, - BLOCK_HEADS: tl.constexpr, - IS_HIP: tl.constexpr, -): - # Keep token count dynamic to avoid compiling every prefill batch length. - heads = tl.program_id(0) * BLOCK_HEADS + tl.arange(0, BLOCK_HEADS) - row, head = heads // H, heads % H - r = tl.arange(0, 512) - # The padded output can exceed 2 GiB at the 64K prefill ceiling. - x_offset = row[:, None].to(tl.int64) * SX + head[:, None] * 512 + r[None, :] - value = tl.load(X + x_offset, row[:, None] < M, 0).to(tl.float32) - partner = tl.gather(value, tl.broadcast_to((r ^ 1)[None, :], (BLOCK_HEADS, 512)), 1) - position = tl.load(POS + row, row < M, 0) - freq_offset = position[:, None].to(tl.int64) * 64 + (r[None, :] - 448) // 2 * 2 - rope_mask = (row[:, None] < M) & (r[None, :] >= 448) - cos = tl.load(F + freq_offset, rope_mask, 0) - sin = tl.load(F + freq_offset + 1, rope_mask, 0) - # Keep the same arithmetic and BF16 rounding as the decode kernel. - even = tl.fma(value, cos, -partner * sin) - if IS_HIP: - # Match AMD flat RoPE: round the sine product before the cosine FMA. - odd = tl.fma(value, cos, partner * sin) - else: - odd = tl.fma(partner, sin, value * cos) - rotated = tl.where((r[None, :] & 1) == 0, even, odd) - y_offset = row[:, None].to(tl.int64) * SY + head[:, None] * 512 + r[None, :] - tl.store( - Y + y_offset, - tl.where(r[None, :] >= 448, rotated, value), - row[:, None] < M, - ) - - -def q_rope_store( - q: torch.Tensor, - output: torch.Tensor, - freqs_cis: torch.Tensor, - positions: torch.Tensor, -) -> None: - """Apply 64-wide forward RoPE to 512-wide heads without changing Q padding.""" - assert q.shape == output.shape and q.ndim == 3 and q.shape[2] == 512 - assert q.dtype == output.dtype == torch.bfloat16 - assert q.stride(2) == output.stride(2) == 1 - assert q.stride(1) == output.stride(1) == 512 - assert freqs_cis.dtype == torch.complex64 and freqs_cis.is_contiguous() - assert freqs_cis.shape[1] == 32 and positions.shape == (q.shape[0],) - assert positions.dtype in (torch.int32, torch.int64) and positions.is_contiguous() - if q.shape[0] >= 4096 and q.shape[1] == 16: - _q_rope_store_prefill[(triton.cdiv(q.shape[0] * q.shape[1], 4),)]( - q, - output, - torch.view_as_real(freqs_cis), - positions, - q.shape[0], - q.shape[1], - q.stride(0), - output.stride(0), - BLOCK_HEADS=4, - IS_HIP=is_hip(), - num_warps=4, - ) - return - pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if is_arch_support_pdl() else {} - _q_rope_store[(q.shape[0], q.shape[1])]( - q, - output, - torch.view_as_real(freqs_cis), - positions, - q.stride(0), - output.stride(0), - IS_HIP=is_hip(), - num_warps=4, - **pdl_kwargs, - ) diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 1177c0910c..71192f1962 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -1341,27 +1341,15 @@ class MQALayer(MqaAttentionBase): q, _ = self.wq_b(q) q = q.view(-1, self.n_local_heads, self.head_dim) if not self.q_head_norm: - if ( - (_is_cuda or _is_gfx95_supported) - and q_out is not None - and ( - 0 < q.shape[0] <= 8 - or ( - self.is_dsv41 - and (get_platform().is_blackwell or _is_gfx95_supported) - and self.n_local_heads == 16 - and 4096 <= q.shape[0] <= 65536 - ) + # TODO: enable zero-copy path to skip the extra no-rope copy overhead + if (_is_cuda or _is_gfx95_supported) and q_out is not None: + fused_q_norm_rope( + q, + q_out, + None, # eps = None means no norm + self.freqs_cis, + positions, ) - and self.head_dim == 512 - and self.qk_rope_head_dim == 64 - and q.dtype == q_out.dtype == torch.bfloat16 - and q.stride(1) == q_out.stride(1) == 512 - and q.stride(2) == q_out.stride(2) == 1 - ): - from sglang.kernels.ops.attention.dsv4.q_rope_store import q_rope_store - - q_rope_store(q, q_out, self.freqs_cis, positions) return q_out fused_rope_inplace( q[..., -self.qk_rope_head_dim :], diff --git a/test/registered/kernels/ops/attention/test_dsv4_q_rope_store.py b/test/registered/kernels/ops/attention/test_dsv4_q_rope_store.py index 3f34f84ccc..1b418bca40 100644 --- a/test/registered/kernels/ops/attention/test_dsv4_q_rope_store.py +++ b/test/registered/kernels/ops/attention/test_dsv4_q_rope_store.py @@ -3,14 +3,16 @@ import unittest import torch from sglang.kernels.ops.attention import deepseek_v4_rope -from sglang.kernels.ops.attention.dsv4.elementwise import fused_rope_inplace -from sglang.kernels.ops.attention.dsv4.q_rope_store import q_rope_store +from sglang.kernels.ops.attention.dsv4.elementwise import ( + fused_q_norm_rope, + fused_rope_inplace, +) from sglang.srt.utils import is_hip from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=40, stage="base-b-kernel-unit", runner_config="1-gpu-large") -# gfx950: q_rope_store's HIP operation order matches the batched RoPE DeepseekV4 turns on +# gfx950: fused_q_norm_rope's HIP operation order matches the batched RoPE DeepseekV4 turns on register_amd_ci(est_time=40, stage="stage-b", runner_config="1-gpu-small-amd-mi35x") @@ -26,7 +28,7 @@ class TestQRopeStore(CustomTestCase): freqs = torch.polar( torch.ones(8192, 32, device="cuda"), torch.randn(8192, 32, device="cuda") ) - for rows in (1, 2, 5, 6, 8): + for rows in (1, 2, 5, 6, 8, 9, 100, 4095): for heads in (8, 16, 32): for dtype in (torch.int32, torch.int64): q = torch.randn( @@ -42,7 +44,7 @@ class TestQRopeStore(CustomTestCase): original = q.clone() expected = q.clone() fused_rope_inplace(expected[..., 448:], None, freqs, positions) - q_rope_store(q, output, freqs, positions) + fused_q_norm_rope(q, output, None, freqs, positions) torch.testing.assert_close(output, expected, rtol=0, atol=0) torch.testing.assert_close(q, original, rtol=0, atol=0) self.assertTrue((padding[:, heads:] == 7).all().item()) @@ -67,7 +69,7 @@ class TestQRopeStore(CustomTestCase): 0, 8192, (rows,), device="cuda", dtype=dtype ) fused_rope_inplace(expected[..., 448:], None, freqs, positions) - q_rope_store(q, padding[:, :16], freqs, positions) + fused_q_norm_rope(q, padding[:, :16], None, freqs, positions) torch.testing.assert_close( padding[:, :16], expected, rtol=0, atol=0 ) @@ -81,10 +83,10 @@ class TestQRopeStore(CustomTestCase): torch.ones(8192, 32, device="cuda"), torch.randn(8192, 32, device="cuda") ) positions = torch.arange(rows, device="cuda") % 8192 - q_rope_store(q, output, freqs, positions) + fused_q_norm_rope(q, output, None, freqs, positions) graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph): - q_rope_store(q, output, freqs, positions) + fused_q_norm_rope(q, output, None, freqs, positions) for _ in range(3): q.normal_() positions.random_(0, 8192)