[DSv4.1] Fold q_rope_store into fused_q_norm_rope (#41657)

Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
(cherry picked from commit 379ec90fb2)
This commit is contained in:
DarkSharpness
2026-10-19 10:15:45 +08:00
committed by minke.yu
parent ecaaa37cb0
commit ad2acaa2df
5 changed files with 64 additions and 177 deletions
@@ -80,7 +80,7 @@ struct FusedQNormRopeParams {
float eps;
};
template <typename DType, int64_t kHeadDim, int64_t kRopeDim, typename PosT, bool kUsePDL>
template <typename DType, int64_t kHeadDim, int64_t kRopeDim, typename PosT, bool kUsePDL, bool kApplyNorm>
Q_KERNEL void fused_q_norm_rope(const __grid_constant__ FusedQNormRopeParams params) {
using namespace device;
@@ -129,24 +129,26 @@ Q_KERNEL void fused_q_norm_rope(const __grid_constant__ FusedQNormRopeParams par
const auto freq = mem_freq.load(params.freqs_cis + position * kRopeDim);
float sum_of_squares = 0.0f;
if constexpr (kApplyNorm) {
float sum_of_squares = 0.0f;
#pragma unroll
for (int i = 0; i < kLocalSize; ++i) {
for (int i = 0; i < kLocalSize; ++i) {
#pragma unroll
for (int j = 0; j < kVecSize; ++j) {
const auto x = cast<float>(input_vec[i][j]);
sum_of_squares += x * x;
for (int j = 0; j < kVecSize; ++j) {
const auto x = cast<float>(input_vec[i][j]);
sum_of_squares += x * x;
}
}
}
sum_of_squares = warp::reduce_sum(sum_of_squares);
const auto norm_factor = math::rsqrt(sum_of_squares / kHeadDim + params.eps);
sum_of_squares = warp::reduce_sum(sum_of_squares);
const auto norm_factor = math::rsqrt(sum_of_squares / kHeadDim + params.eps);
#pragma unroll
for (int i = 0; i < kLocalSize; ++i) {
for (int i = 0; i < kLocalSize; ++i) {
#pragma unroll
for (int j = 0; j < kVecSize; ++j) {
const auto x = cast<float>(input_vec[i][j]);
input_vec[i][j] = cast<DType>(x * norm_factor);
for (int j = 0; j < kVecSize; ++j) {
const auto x = cast<float>(input_vec[i][j]);
input_vec[i][j] = cast<DType>(x * norm_factor);
}
}
}
@@ -172,17 +174,34 @@ Q_KERNEL void fused_q_norm_rope(const __grid_constant__ FusedQNormRopeParams par
const auto elem = mem_elem.load(s_rope[warp_id]);
const auto [x_real, x_imag] = cast<fp32x2_t>(elem);
const auto [freq_real, freq_imag] = freq;
const fp32x2_t rotated = {
x_real * freq_real - x_imag * freq_imag,
x_real * freq_imag + x_imag * freq_real,
};
#ifdef USE_ROCM
// gfx950 batched RoPE rounds the sine product, then one fma with the cosine. Only the
// kApplyNorm = false specialization replaces a kernel that rounded that way, so only it
// matches it; the norm path keeps the order it has always had.
constexpr bool kBatchedRopeOrder = !kApplyNorm;
#else
constexpr bool kBatchedRopeOrder = false;
#endif
fp32x2_t rotated;
if constexpr (kBatchedRopeOrder) {
rotated = {
fmaf(x_real, freq_real, -__fmul_rn(x_imag, freq_imag)),
fmaf(x_imag, freq_real, __fmul_rn(x_real, freq_imag)),
};
} else {
rotated = {
x_real * freq_real - x_imag * freq_imag,
x_real * freq_imag + x_imag * freq_real,
};
}
mem_elem.store(output_ptr + (kHeadDim - kRopeDim), cast<DType2>(rotated));
}
template <typename DType, int64_t kHeadDim, int64_t kRopeDim, bool kUsePDL>
// kApplyNorm = false only rotates the rope tail while copying Q into a (padded) output.
template <typename DType, int64_t kHeadDim, int64_t kRopeDim, bool kApplyNorm, bool kUsePDL>
struct FusedQNormRopeKernel {
template <typename PosT>
static constexpr auto kernel = fused_q_norm_rope<DType, kHeadDim, kRopeDim, PosT, kUsePDL>;
static constexpr auto kernel = fused_q_norm_rope<DType, kHeadDim, kRopeDim, PosT, kUsePDL, kApplyNorm>;
static void forward(
const tvm::ffi::TensorView q_input,
@@ -37,9 +37,10 @@ def _jit_main_q_norm_rope_module(
dtype: torch.dtype,
head_dim: int,
rope_dim: int,
apply_norm: bool,
):
"""Main MLA path Q kernel: rmsnorm-self + RoPE, warp per (token, head)."""
args = make_cpp_args(dtype, head_dim, rope_dim, is_arch_support_pdl())
args = make_cpp_args(dtype, head_dim, rope_dim, apply_norm, is_arch_support_pdl())
return load_jit(
make_name("main_q_norm_rope"),
*args,
@@ -170,7 +171,7 @@ def fused_rope_inplace(
def fused_q_norm_rope(
q_input: torch.Tensor,
q_output: torch.Tensor,
eps: float,
eps: Optional[float],
freqs_cis: torch.Tensor,
positions: torch.Tensor,
) -> None:
@@ -178,10 +179,13 @@ def fused_q_norm_rope(
head_dim = q_input.shape[-1]
rope_dim = freqs_real.shape[-1]
if _is_xpu:
assert eps is not None
fused_q_norm_rope_xpu(q_input, q_output, freqs_real, positions, eps)
else:
module = _jit_main_q_norm_rope_module(q_input.dtype, head_dim, rope_dim)
module.forward(q_input, q_output, freqs_real, positions, eps)
module = _jit_main_q_norm_rope_module(
q_input.dtype, head_dim, rope_dim, eps is not None
)
module.forward(q_input, q_output, freqs_real, positions, eps or 0.0)
def fused_q_indexer_rope_hadamard_quant(
@@ -1,126 +0,0 @@
"""Rotate Q while writing the padded buffer consumed by sparse attention."""
import torch
import triton
import triton.language as tl
from sglang.kernels.jit.utils import is_arch_support_pdl
from sglang.srt.utils import is_hip
@triton.jit
def _q_rope_store(
X,
Y,
F,
POS,
SX: tl.constexpr,
SY: tl.constexpr,
IS_HIP: tl.constexpr,
USE_GDC: tl.constexpr = False,
):
row, head = tl.program_id(0), tl.program_id(1)
r = tl.arange(0, 512)
if USE_GDC:
tl.extra.cuda.gdc_wait() # X is the wq_b GEMM output; POS may be in-graph
value = tl.load(X + row * SX + head * 512 + r).to(tl.float32)
partner = tl.gather(value, r ^ 1, 0)
position = tl.load(POS + row)
cos = tl.load(F + position * 64 + (r - 448) // 2 * 2, r >= 448, 0)
sin = tl.load(F + position * 64 + (r - 448) // 2 * 2 + 1, r >= 448, 0)
# Match the operation order of deepseek_rope_kernel before BF16 rounding.
even = tl.fma(value, cos, -partner * sin)
if IS_HIP:
# Match AMD flat RoPE: round the sine product before the cosine FMA.
odd = tl.fma(value, cos, partner * sin)
else:
odd = tl.fma(partner, sin, value * cos)
rotated = tl.where((r & 1) == 0, even, odd)
tl.store(Y + row * SY + head * 512 + r, tl.where(r >= 448, rotated, value))
if USE_GDC:
tl.extra.cuda.gdc_launch_dependents()
@triton.jit
def _q_rope_store_prefill(
X,
Y,
F,
POS,
M,
H: tl.constexpr,
SX: tl.constexpr,
SY: tl.constexpr,
BLOCK_HEADS: tl.constexpr,
IS_HIP: tl.constexpr,
):
# Keep token count dynamic to avoid compiling every prefill batch length.
heads = tl.program_id(0) * BLOCK_HEADS + tl.arange(0, BLOCK_HEADS)
row, head = heads // H, heads % H
r = tl.arange(0, 512)
# The padded output can exceed 2 GiB at the 64K prefill ceiling.
x_offset = row[:, None].to(tl.int64) * SX + head[:, None] * 512 + r[None, :]
value = tl.load(X + x_offset, row[:, None] < M, 0).to(tl.float32)
partner = tl.gather(value, tl.broadcast_to((r ^ 1)[None, :], (BLOCK_HEADS, 512)), 1)
position = tl.load(POS + row, row < M, 0)
freq_offset = position[:, None].to(tl.int64) * 64 + (r[None, :] - 448) // 2 * 2
rope_mask = (row[:, None] < M) & (r[None, :] >= 448)
cos = tl.load(F + freq_offset, rope_mask, 0)
sin = tl.load(F + freq_offset + 1, rope_mask, 0)
# Keep the same arithmetic and BF16 rounding as the decode kernel.
even = tl.fma(value, cos, -partner * sin)
if IS_HIP:
# Match AMD flat RoPE: round the sine product before the cosine FMA.
odd = tl.fma(value, cos, partner * sin)
else:
odd = tl.fma(partner, sin, value * cos)
rotated = tl.where((r[None, :] & 1) == 0, even, odd)
y_offset = row[:, None].to(tl.int64) * SY + head[:, None] * 512 + r[None, :]
tl.store(
Y + y_offset,
tl.where(r[None, :] >= 448, rotated, value),
row[:, None] < M,
)
def q_rope_store(
q: torch.Tensor,
output: torch.Tensor,
freqs_cis: torch.Tensor,
positions: torch.Tensor,
) -> None:
"""Apply 64-wide forward RoPE to 512-wide heads without changing Q padding."""
assert q.shape == output.shape and q.ndim == 3 and q.shape[2] == 512
assert q.dtype == output.dtype == torch.bfloat16
assert q.stride(2) == output.stride(2) == 1
assert q.stride(1) == output.stride(1) == 512
assert freqs_cis.dtype == torch.complex64 and freqs_cis.is_contiguous()
assert freqs_cis.shape[1] == 32 and positions.shape == (q.shape[0],)
assert positions.dtype in (torch.int32, torch.int64) and positions.is_contiguous()
if q.shape[0] >= 4096 and q.shape[1] == 16:
_q_rope_store_prefill[(triton.cdiv(q.shape[0] * q.shape[1], 4),)](
q,
output,
torch.view_as_real(freqs_cis),
positions,
q.shape[0],
q.shape[1],
q.stride(0),
output.stride(0),
BLOCK_HEADS=4,
IS_HIP=is_hip(),
num_warps=4,
)
return
pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if is_arch_support_pdl() else {}
_q_rope_store[(q.shape[0], q.shape[1])](
q,
output,
torch.view_as_real(freqs_cis),
positions,
q.stride(0),
output.stride(0),
IS_HIP=is_hip(),
num_warps=4,
**pdl_kwargs,
)
+8 -20
View File
@@ -1309,27 +1309,15 @@ class MQALayer(MqaAttentionBase):
q, _ = self.wq_b(q)
q = q.view(-1, self.n_local_heads, self.head_dim)
if not self.q_head_norm:
if (
_is_cuda
and q_out is not None
and (
0 < q.shape[0] <= 8
or (
self.is_dsv41
and get_platform().is_blackwell
and self.n_local_heads == 16
and 4096 <= q.shape[0] <= 65536
)
# TODO: enable zero-copy path to skip the extra no-rope copy overhead
if (_is_cuda or _is_gfx95_supported) and q_out is not None:
fused_q_norm_rope(
q,
q_out,
None, # eps = None means no norm
self.freqs_cis,
positions,
)
and self.head_dim == 512
and self.qk_rope_head_dim == 64
and q.dtype == q_out.dtype == torch.bfloat16
and q.stride(1) == q_out.stride(1) == 512
and q.stride(2) == q_out.stride(2) == 1
):
from sglang.kernels.ops.attention.dsv4.q_rope_store import q_rope_store
q_rope_store(q, q_out, self.freqs_cis, positions)
return q_out
fused_rope_inplace(
q[..., -self.qk_rope_head_dim :],