[Fix] [AMD] qwen3: Fix KV cache write in the fused QK-norm + mRoPE decode path (#44266)

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
jiaqiang-dot-liu
2026-10-15 21:22:46 +08:00
committed by GitHub
co-authored by Cursor
parent 2092160d42
commit 2f5407da28
3 changed files with 243 additions and 53 deletions
@@ -86,6 +86,9 @@ if _is_xpu:
class RotaryEmbedding(BaseFusedOp):
"""Original rotary positional embedding."""
# (source cos_sin_cache, cast copy) memo for cos_sin_cache_as().
_cos_sin_cache_cast: Optional[Tuple[torch.Tensor, torch.Tensor]] = None
def __init__(
self,
head_size: int,
@@ -214,6 +217,19 @@ class RotaryEmbedding(BaseFusedOp):
query.device, dtype=query.dtype
)
def cos_sin_cache_as(self, dtype: torch.dtype) -> torch.Tensor:
"""cos_sin_cache in ``dtype``, for kernels that read it in the activation
dtype. The cast copy is reused until cos_sin_cache is replaced (growth or
device move), so steady-state calls launch no cast kernel."""
cache = self.cos_sin_cache
if cache.dtype == dtype:
return cache
memo = self._cos_sin_cache_cast
if memo is None or memo[0] is not cache or memo[1].dtype != dtype:
memo = (cache, cache.to(dtype))
self._cos_sin_cache_cast = memo
return memo[1]
def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
"""Compute the inverse frequency."""
# NOTE(woosuk): To exactly match the HF implementation, we need to
+81 -53
View File
@@ -65,6 +65,65 @@ if _is_npu:
from sglang.srt.hardware_backend.npu.cmo import get_cmo_stream, wait_cmo_stream
def aiter_fused_qk_norm_rope_set_kv(
qkv: torch.Tensor,
q_norm: RMSNorm,
k_norm: RMSNorm,
rotary_emb: MRotaryEmbedding,
positions: torch.Tensor,
k_cache: torch.Tensor,
v_cache: torch.Tensor,
out_cache_loc: torch.Tensor,
num_heads: int,
num_kv_heads: int,
head_dim: int,
k_scale: torch.Tensor,
v_scale: torch.Tensor,
) -> torch.Tensor:
"""QK-norm + 3D mRoPE + KV cache write in one aiter HIP kernel.
``qkv`` is the (num_tokens, (num_heads + 2 * num_kv_heads) * head_dim) QKV
projection; ``k_cache`` / ``v_cache`` are the NHD (slots, num_kv_heads,
head_dim) pool buffers. ``k_scale`` / ``v_scale`` must be CPU tensors: the
kernel reads them on the host. Returns q as (num_tokens, num_heads * head_dim).
"""
num_tokens = qkv.shape[0]
q_out = torch.empty(
num_tokens, num_heads, head_dim, dtype=qkv.dtype, device=qkv.device
)
# The kernel reads cache strides as (block, token, head); an NHD buffer is
# one-slot blocks addressed with block_size=0.
fused_qk_norm_mrope_3d_cache_pts_quant_shuffle(
qkv.view(num_tokens, -1, head_dim),
q_norm.weight,
k_norm.weight,
rotary_emb.cos_sin_cache_as(qkv.dtype),
positions,
num_tokens,
num_heads,
num_kv_heads,
num_kv_heads,
head_dim,
rotary_emb.is_neox_style,
rotary_emb.mrope_section,
rotary_emb.mrope_interleaved,
q_norm.variance_epsilon,
q_out,
k_cache.unsqueeze(1),
v_cache.unsqueeze(1),
out_cache_loc,
k_scale,
v_scale,
None,
None,
False,
False,
0,
0,
)
return q_out.view(num_tokens, -1)
class Qwen3Attention(nn.Module):
def __init__(
self,
@@ -204,65 +263,26 @@ class Qwen3Attention(nn.Module):
return q, k, v
def forward_prepare_aiter_fused_mrope(
self, positions, hidden_states, forward_batch
self, positions, hidden_states, forward_batch, k_cache, v_cache
):
"""Fused QK-norm + 3D mRoPE + KV cache write for decode (ROCm/aiter).
The fused HIP kernel replaces split → QK norm → mRoPE → cache write,
so KV is already in the paged cache when this returns.
Returns (q, None, None); caller must pass save_kv_cache=False to attn.
"""
"""Decode-only fused path; KV is already in the paged cache when this
returns, so the caller must pass save_kv_cache=False to attn."""
qkv, _ = self.qkv_proj(hidden_states)
num_tokens = qkv.shape[0]
qkv_3d = qkv.view(num_tokens, -1, self.head_dim)
token_to_kv_pool = get_token_to_kv_pool()
k_cache, v_cache = token_to_kv_pool.get_kv_buffer(self.attn.layer_id)
slot_mapping = forward_batch.out_cache_loc
cos_sin = self.rotary_emb.cos_sin_cache
if cos_sin.dtype != qkv.dtype:
cos_sin = cos_sin.to(dtype=qkv.dtype)
q_out = torch.empty(
num_tokens,
self.num_heads,
self.head_dim,
dtype=qkv.dtype,
device=qkv.device,
)
fused_qk_norm_mrope_3d_cache_pts_quant_shuffle(
qkv_3d,
self.q_norm.weight,
self.k_norm.weight,
cos_sin,
q = aiter_fused_qk_norm_rope_set_kv(
qkv,
self.q_norm,
self.k_norm,
self.rotary_emb,
positions,
num_tokens,
self.num_heads,
self.num_kv_heads,
self.num_kv_heads,
self.head_dim,
self.rotary_emb.is_neox_style,
self.rotary_emb.mrope_section,
self.rotary_emb.mrope_interleaved,
self.q_norm.variance_epsilon,
q_out,
k_cache,
v_cache,
slot_mapping,
forward_batch.out_cache_loc,
self.num_heads,
self.num_kv_heads,
self.head_dim,
self._fused_k_scale,
self._fused_v_scale,
None,
None,
False,
False,
0,
0,
)
q = q_out.reshape(num_tokens, -1)
return q, None, None
def forward(
@@ -275,15 +295,23 @@ class Qwen3Attention(nn.Module):
hidden_states = hidden_states.bfloat16()
save_kv_cache = True
use_aiter_fused = (
use_aiter_fused = False
can_try_aiter_fused = (
self.use_fused_qk_norm_mrope
and forward_batch.forward_mode.is_decode()
and get_exec().deterministic.rl_on_policy_target is None
)
if can_try_aiter_fused:
k_cache, v_cache = get_token_to_kv_pool().get_kv_buffer(self.attn.layer_id)
# The fused write targets the NHD layout in the activation dtype and
# does not apply per-layer KV scales.
use_aiter_fused = (
k_cache.dim() == 3 and k_cache.dtype == hidden_states.dtype
)
if use_aiter_fused:
q, k, v = self.forward_prepare_aiter_fused_mrope(
positions, hidden_states, forward_batch
positions, hidden_states, forward_batch, k_cache, v_cache
)
save_kv_cache = False
elif not _is_npu: