[DSv4.1] Fold q_rope_store into fused_q_norm_rope (#41657)
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
7be5e3473c
commit
379ec90fb2
@@ -80,7 +80,7 @@ struct FusedQNormRopeParams {
|
||||
float eps;
|
||||
};
|
||||
|
||||
template <typename DType, int64_t kHeadDim, int64_t kRopeDim, typename PosT, bool kUsePDL>
|
||||
template <typename DType, int64_t kHeadDim, int64_t kRopeDim, typename PosT, bool kUsePDL, bool kApplyNorm>
|
||||
Q_KERNEL void fused_q_norm_rope(const __grid_constant__ FusedQNormRopeParams params) {
|
||||
using namespace device;
|
||||
|
||||
@@ -129,24 +129,26 @@ Q_KERNEL void fused_q_norm_rope(const __grid_constant__ FusedQNormRopeParams par
|
||||
|
||||
const auto freq = mem_freq.load(params.freqs_cis + position * kRopeDim);
|
||||
|
||||
float sum_of_squares = 0.0f;
|
||||
if constexpr (kApplyNorm) {
|
||||
float sum_of_squares = 0.0f;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kLocalSize; ++i) {
|
||||
for (int i = 0; i < kLocalSize; ++i) {
|
||||
#pragma unroll
|
||||
for (int j = 0; j < kVecSize; ++j) {
|
||||
const auto x = cast<float>(input_vec[i][j]);
|
||||
sum_of_squares += x * x;
|
||||
for (int j = 0; j < kVecSize; ++j) {
|
||||
const auto x = cast<float>(input_vec[i][j]);
|
||||
sum_of_squares += x * x;
|
||||
}
|
||||
}
|
||||
}
|
||||
sum_of_squares = warp::reduce_sum(sum_of_squares);
|
||||
const auto norm_factor = math::rsqrt(sum_of_squares / kHeadDim + params.eps);
|
||||
sum_of_squares = warp::reduce_sum(sum_of_squares);
|
||||
const auto norm_factor = math::rsqrt(sum_of_squares / kHeadDim + params.eps);
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kLocalSize; ++i) {
|
||||
for (int i = 0; i < kLocalSize; ++i) {
|
||||
#pragma unroll
|
||||
for (int j = 0; j < kVecSize; ++j) {
|
||||
const auto x = cast<float>(input_vec[i][j]);
|
||||
input_vec[i][j] = cast<DType>(x * norm_factor);
|
||||
for (int j = 0; j < kVecSize; ++j) {
|
||||
const auto x = cast<float>(input_vec[i][j]);
|
||||
input_vec[i][j] = cast<DType>(x * norm_factor);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,17 +174,34 @@ Q_KERNEL void fused_q_norm_rope(const __grid_constant__ FusedQNormRopeParams par
|
||||
const auto elem = mem_elem.load(s_rope[warp_id]);
|
||||
const auto [x_real, x_imag] = cast<fp32x2_t>(elem);
|
||||
const auto [freq_real, freq_imag] = freq;
|
||||
const fp32x2_t rotated = {
|
||||
x_real * freq_real - x_imag * freq_imag,
|
||||
x_real * freq_imag + x_imag * freq_real,
|
||||
};
|
||||
#ifdef USE_ROCM
|
||||
// gfx950 batched RoPE rounds the sine product, then one fma with the cosine. Only the
|
||||
// kApplyNorm = false specialization replaces a kernel that rounded that way, so only it
|
||||
// matches it; the norm path keeps the order it has always had.
|
||||
constexpr bool kBatchedRopeOrder = !kApplyNorm;
|
||||
#else
|
||||
constexpr bool kBatchedRopeOrder = false;
|
||||
#endif
|
||||
fp32x2_t rotated;
|
||||
if constexpr (kBatchedRopeOrder) {
|
||||
rotated = {
|
||||
fmaf(x_real, freq_real, -__fmul_rn(x_imag, freq_imag)),
|
||||
fmaf(x_imag, freq_real, __fmul_rn(x_real, freq_imag)),
|
||||
};
|
||||
} else {
|
||||
rotated = {
|
||||
x_real * freq_real - x_imag * freq_imag,
|
||||
x_real * freq_imag + x_imag * freq_real,
|
||||
};
|
||||
}
|
||||
mem_elem.store(output_ptr + (kHeadDim - kRopeDim), cast<DType2>(rotated));
|
||||
}
|
||||
|
||||
template <typename DType, int64_t kHeadDim, int64_t kRopeDim, bool kUsePDL>
|
||||
// kApplyNorm = false only rotates the rope tail while copying Q into a (padded) output.
|
||||
template <typename DType, int64_t kHeadDim, int64_t kRopeDim, bool kApplyNorm, bool kUsePDL>
|
||||
struct FusedQNormRopeKernel {
|
||||
template <typename PosT>
|
||||
static constexpr auto kernel = fused_q_norm_rope<DType, kHeadDim, kRopeDim, PosT, kUsePDL>;
|
||||
static constexpr auto kernel = fused_q_norm_rope<DType, kHeadDim, kRopeDim, PosT, kUsePDL, kApplyNorm>;
|
||||
|
||||
static void forward(
|
||||
const tvm::ffi::TensorView q_input,
|
||||
|
||||
@@ -37,9 +37,10 @@ def _jit_main_q_norm_rope_module(
|
||||
dtype: torch.dtype,
|
||||
head_dim: int,
|
||||
rope_dim: int,
|
||||
apply_norm: bool,
|
||||
):
|
||||
"""Main MLA path Q kernel: rmsnorm-self + RoPE, warp per (token, head)."""
|
||||
args = make_cpp_args(dtype, head_dim, rope_dim, is_arch_support_pdl())
|
||||
args = make_cpp_args(dtype, head_dim, rope_dim, apply_norm, is_arch_support_pdl())
|
||||
return load_jit(
|
||||
make_name("main_q_norm_rope"),
|
||||
*args,
|
||||
@@ -170,7 +171,7 @@ def fused_rope_inplace(
|
||||
def fused_q_norm_rope(
|
||||
q_input: torch.Tensor,
|
||||
q_output: torch.Tensor,
|
||||
eps: float,
|
||||
eps: Optional[float],
|
||||
freqs_cis: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> None:
|
||||
@@ -178,10 +179,13 @@ def fused_q_norm_rope(
|
||||
head_dim = q_input.shape[-1]
|
||||
rope_dim = freqs_real.shape[-1]
|
||||
if _is_xpu:
|
||||
assert eps is not None
|
||||
fused_q_norm_rope_xpu(q_input, q_output, freqs_real, positions, eps)
|
||||
else:
|
||||
module = _jit_main_q_norm_rope_module(q_input.dtype, head_dim, rope_dim)
|
||||
module.forward(q_input, q_output, freqs_real, positions, eps)
|
||||
module = _jit_main_q_norm_rope_module(
|
||||
q_input.dtype, head_dim, rope_dim, eps is not None
|
||||
)
|
||||
module.forward(q_input, q_output, freqs_real, positions, eps or 0.0)
|
||||
|
||||
|
||||
def fused_q_indexer_rope_hadamard_quant(
|
||||
|
||||
@@ -1,126 +0,0 @@
|
||||
"""Rotate Q while writing the padded buffer consumed by sparse attention."""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.jit.utils import is_arch_support_pdl
|
||||
from sglang.srt.utils import is_hip
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _q_rope_store(
|
||||
X,
|
||||
Y,
|
||||
F,
|
||||
POS,
|
||||
SX: tl.constexpr,
|
||||
SY: tl.constexpr,
|
||||
IS_HIP: tl.constexpr,
|
||||
USE_GDC: tl.constexpr = False,
|
||||
):
|
||||
row, head = tl.program_id(0), tl.program_id(1)
|
||||
r = tl.arange(0, 512)
|
||||
if USE_GDC:
|
||||
tl.extra.cuda.gdc_wait() # X is the wq_b GEMM output; POS may be in-graph
|
||||
value = tl.load(X + row * SX + head * 512 + r).to(tl.float32)
|
||||
partner = tl.gather(value, r ^ 1, 0)
|
||||
position = tl.load(POS + row)
|
||||
cos = tl.load(F + position * 64 + (r - 448) // 2 * 2, r >= 448, 0)
|
||||
sin = tl.load(F + position * 64 + (r - 448) // 2 * 2 + 1, r >= 448, 0)
|
||||
# Match the operation order of deepseek_rope_kernel before BF16 rounding.
|
||||
even = tl.fma(value, cos, -partner * sin)
|
||||
if IS_HIP:
|
||||
# Match AMD flat RoPE: round the sine product before the cosine FMA.
|
||||
odd = tl.fma(value, cos, partner * sin)
|
||||
else:
|
||||
odd = tl.fma(partner, sin, value * cos)
|
||||
rotated = tl.where((r & 1) == 0, even, odd)
|
||||
tl.store(Y + row * SY + head * 512 + r, tl.where(r >= 448, rotated, value))
|
||||
if USE_GDC:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _q_rope_store_prefill(
|
||||
X,
|
||||
Y,
|
||||
F,
|
||||
POS,
|
||||
M,
|
||||
H: tl.constexpr,
|
||||
SX: tl.constexpr,
|
||||
SY: tl.constexpr,
|
||||
BLOCK_HEADS: tl.constexpr,
|
||||
IS_HIP: tl.constexpr,
|
||||
):
|
||||
# Keep token count dynamic to avoid compiling every prefill batch length.
|
||||
heads = tl.program_id(0) * BLOCK_HEADS + tl.arange(0, BLOCK_HEADS)
|
||||
row, head = heads // H, heads % H
|
||||
r = tl.arange(0, 512)
|
||||
# The padded output can exceed 2 GiB at the 64K prefill ceiling.
|
||||
x_offset = row[:, None].to(tl.int64) * SX + head[:, None] * 512 + r[None, :]
|
||||
value = tl.load(X + x_offset, row[:, None] < M, 0).to(tl.float32)
|
||||
partner = tl.gather(value, tl.broadcast_to((r ^ 1)[None, :], (BLOCK_HEADS, 512)), 1)
|
||||
position = tl.load(POS + row, row < M, 0)
|
||||
freq_offset = position[:, None].to(tl.int64) * 64 + (r[None, :] - 448) // 2 * 2
|
||||
rope_mask = (row[:, None] < M) & (r[None, :] >= 448)
|
||||
cos = tl.load(F + freq_offset, rope_mask, 0)
|
||||
sin = tl.load(F + freq_offset + 1, rope_mask, 0)
|
||||
# Keep the same arithmetic and BF16 rounding as the decode kernel.
|
||||
even = tl.fma(value, cos, -partner * sin)
|
||||
if IS_HIP:
|
||||
# Match AMD flat RoPE: round the sine product before the cosine FMA.
|
||||
odd = tl.fma(value, cos, partner * sin)
|
||||
else:
|
||||
odd = tl.fma(partner, sin, value * cos)
|
||||
rotated = tl.where((r[None, :] & 1) == 0, even, odd)
|
||||
y_offset = row[:, None].to(tl.int64) * SY + head[:, None] * 512 + r[None, :]
|
||||
tl.store(
|
||||
Y + y_offset,
|
||||
tl.where(r[None, :] >= 448, rotated, value),
|
||||
row[:, None] < M,
|
||||
)
|
||||
|
||||
|
||||
def q_rope_store(
|
||||
q: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
freqs_cis: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> None:
|
||||
"""Apply 64-wide forward RoPE to 512-wide heads without changing Q padding."""
|
||||
assert q.shape == output.shape and q.ndim == 3 and q.shape[2] == 512
|
||||
assert q.dtype == output.dtype == torch.bfloat16
|
||||
assert q.stride(2) == output.stride(2) == 1
|
||||
assert q.stride(1) == output.stride(1) == 512
|
||||
assert freqs_cis.dtype == torch.complex64 and freqs_cis.is_contiguous()
|
||||
assert freqs_cis.shape[1] == 32 and positions.shape == (q.shape[0],)
|
||||
assert positions.dtype in (torch.int32, torch.int64) and positions.is_contiguous()
|
||||
if q.shape[0] >= 4096 and q.shape[1] == 16:
|
||||
_q_rope_store_prefill[(triton.cdiv(q.shape[0] * q.shape[1], 4),)](
|
||||
q,
|
||||
output,
|
||||
torch.view_as_real(freqs_cis),
|
||||
positions,
|
||||
q.shape[0],
|
||||
q.shape[1],
|
||||
q.stride(0),
|
||||
output.stride(0),
|
||||
BLOCK_HEADS=4,
|
||||
IS_HIP=is_hip(),
|
||||
num_warps=4,
|
||||
)
|
||||
return
|
||||
pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if is_arch_support_pdl() else {}
|
||||
_q_rope_store[(q.shape[0], q.shape[1])](
|
||||
q,
|
||||
output,
|
||||
torch.view_as_real(freqs_cis),
|
||||
positions,
|
||||
q.stride(0),
|
||||
output.stride(0),
|
||||
IS_HIP=is_hip(),
|
||||
num_warps=4,
|
||||
**pdl_kwargs,
|
||||
)
|
||||
@@ -1341,27 +1341,15 @@ class MQALayer(MqaAttentionBase):
|
||||
q, _ = self.wq_b(q)
|
||||
q = q.view(-1, self.n_local_heads, self.head_dim)
|
||||
if not self.q_head_norm:
|
||||
if (
|
||||
(_is_cuda or _is_gfx95_supported)
|
||||
and q_out is not None
|
||||
and (
|
||||
0 < q.shape[0] <= 8
|
||||
or (
|
||||
self.is_dsv41
|
||||
and (get_platform().is_blackwell or _is_gfx95_supported)
|
||||
and self.n_local_heads == 16
|
||||
and 4096 <= q.shape[0] <= 65536
|
||||
)
|
||||
# TODO: enable zero-copy path to skip the extra no-rope copy overhead
|
||||
if (_is_cuda or _is_gfx95_supported) and q_out is not None:
|
||||
fused_q_norm_rope(
|
||||
q,
|
||||
q_out,
|
||||
None, # eps = None means no norm
|
||||
self.freqs_cis,
|
||||
positions,
|
||||
)
|
||||
and self.head_dim == 512
|
||||
and self.qk_rope_head_dim == 64
|
||||
and q.dtype == q_out.dtype == torch.bfloat16
|
||||
and q.stride(1) == q_out.stride(1) == 512
|
||||
and q.stride(2) == q_out.stride(2) == 1
|
||||
):
|
||||
from sglang.kernels.ops.attention.dsv4.q_rope_store import q_rope_store
|
||||
|
||||
q_rope_store(q, q_out, self.freqs_cis, positions)
|
||||
return q_out
|
||||
fused_rope_inplace(
|
||||
q[..., -self.qk_rope_head_dim :],
|
||||
|
||||
Reference in New Issue
Block a user