[Fix] [AMD] qwen3: Fix KV cache write in the fused QK-norm + mRoPE decode path (#44266)
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
2092160d42
commit
2f5407da28
@@ -86,6 +86,9 @@ if _is_xpu:
|
||||
class RotaryEmbedding(BaseFusedOp):
|
||||
"""Original rotary positional embedding."""
|
||||
|
||||
# (source cos_sin_cache, cast copy) memo for cos_sin_cache_as().
|
||||
_cos_sin_cache_cast: Optional[Tuple[torch.Tensor, torch.Tensor]] = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
head_size: int,
|
||||
@@ -214,6 +217,19 @@ class RotaryEmbedding(BaseFusedOp):
|
||||
query.device, dtype=query.dtype
|
||||
)
|
||||
|
||||
def cos_sin_cache_as(self, dtype: torch.dtype) -> torch.Tensor:
|
||||
"""cos_sin_cache in ``dtype``, for kernels that read it in the activation
|
||||
dtype. The cast copy is reused until cos_sin_cache is replaced (growth or
|
||||
device move), so steady-state calls launch no cast kernel."""
|
||||
cache = self.cos_sin_cache
|
||||
if cache.dtype == dtype:
|
||||
return cache
|
||||
memo = self._cos_sin_cache_cast
|
||||
if memo is None or memo[0] is not cache or memo[1].dtype != dtype:
|
||||
memo = (cache, cache.to(dtype))
|
||||
self._cos_sin_cache_cast = memo
|
||||
return memo[1]
|
||||
|
||||
def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
|
||||
"""Compute the inverse frequency."""
|
||||
# NOTE(woosuk): To exactly match the HF implementation, we need to
|
||||
|
||||
@@ -65,6 +65,65 @@ if _is_npu:
|
||||
from sglang.srt.hardware_backend.npu.cmo import get_cmo_stream, wait_cmo_stream
|
||||
|
||||
|
||||
def aiter_fused_qk_norm_rope_set_kv(
|
||||
qkv: torch.Tensor,
|
||||
q_norm: RMSNorm,
|
||||
k_norm: RMSNorm,
|
||||
rotary_emb: MRotaryEmbedding,
|
||||
positions: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
v_cache: torch.Tensor,
|
||||
out_cache_loc: torch.Tensor,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
k_scale: torch.Tensor,
|
||||
v_scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""QK-norm + 3D mRoPE + KV cache write in one aiter HIP kernel.
|
||||
|
||||
``qkv`` is the (num_tokens, (num_heads + 2 * num_kv_heads) * head_dim) QKV
|
||||
projection; ``k_cache`` / ``v_cache`` are the NHD (slots, num_kv_heads,
|
||||
head_dim) pool buffers. ``k_scale`` / ``v_scale`` must be CPU tensors: the
|
||||
kernel reads them on the host. Returns q as (num_tokens, num_heads * head_dim).
|
||||
"""
|
||||
num_tokens = qkv.shape[0]
|
||||
q_out = torch.empty(
|
||||
num_tokens, num_heads, head_dim, dtype=qkv.dtype, device=qkv.device
|
||||
)
|
||||
# The kernel reads cache strides as (block, token, head); an NHD buffer is
|
||||
# one-slot blocks addressed with block_size=0.
|
||||
fused_qk_norm_mrope_3d_cache_pts_quant_shuffle(
|
||||
qkv.view(num_tokens, -1, head_dim),
|
||||
q_norm.weight,
|
||||
k_norm.weight,
|
||||
rotary_emb.cos_sin_cache_as(qkv.dtype),
|
||||
positions,
|
||||
num_tokens,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
rotary_emb.is_neox_style,
|
||||
rotary_emb.mrope_section,
|
||||
rotary_emb.mrope_interleaved,
|
||||
q_norm.variance_epsilon,
|
||||
q_out,
|
||||
k_cache.unsqueeze(1),
|
||||
v_cache.unsqueeze(1),
|
||||
out_cache_loc,
|
||||
k_scale,
|
||||
v_scale,
|
||||
None,
|
||||
None,
|
||||
False,
|
||||
False,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
return q_out.view(num_tokens, -1)
|
||||
|
||||
|
||||
class Qwen3Attention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -204,65 +263,26 @@ class Qwen3Attention(nn.Module):
|
||||
return q, k, v
|
||||
|
||||
def forward_prepare_aiter_fused_mrope(
|
||||
self, positions, hidden_states, forward_batch
|
||||
self, positions, hidden_states, forward_batch, k_cache, v_cache
|
||||
):
|
||||
"""Fused QK-norm + 3D mRoPE + KV cache write for decode (ROCm/aiter).
|
||||
|
||||
The fused HIP kernel replaces split → QK norm → mRoPE → cache write,
|
||||
so KV is already in the paged cache when this returns.
|
||||
Returns (q, None, None); caller must pass save_kv_cache=False to attn.
|
||||
"""
|
||||
"""Decode-only fused path; KV is already in the paged cache when this
|
||||
returns, so the caller must pass save_kv_cache=False to attn."""
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
num_tokens = qkv.shape[0]
|
||||
|
||||
qkv_3d = qkv.view(num_tokens, -1, self.head_dim)
|
||||
|
||||
token_to_kv_pool = get_token_to_kv_pool()
|
||||
k_cache, v_cache = token_to_kv_pool.get_kv_buffer(self.attn.layer_id)
|
||||
slot_mapping = forward_batch.out_cache_loc
|
||||
|
||||
cos_sin = self.rotary_emb.cos_sin_cache
|
||||
if cos_sin.dtype != qkv.dtype:
|
||||
cos_sin = cos_sin.to(dtype=qkv.dtype)
|
||||
|
||||
q_out = torch.empty(
|
||||
num_tokens,
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
dtype=qkv.dtype,
|
||||
device=qkv.device,
|
||||
)
|
||||
|
||||
fused_qk_norm_mrope_3d_cache_pts_quant_shuffle(
|
||||
qkv_3d,
|
||||
self.q_norm.weight,
|
||||
self.k_norm.weight,
|
||||
cos_sin,
|
||||
q = aiter_fused_qk_norm_rope_set_kv(
|
||||
qkv,
|
||||
self.q_norm,
|
||||
self.k_norm,
|
||||
self.rotary_emb,
|
||||
positions,
|
||||
num_tokens,
|
||||
self.num_heads,
|
||||
self.num_kv_heads,
|
||||
self.num_kv_heads,
|
||||
self.head_dim,
|
||||
self.rotary_emb.is_neox_style,
|
||||
self.rotary_emb.mrope_section,
|
||||
self.rotary_emb.mrope_interleaved,
|
||||
self.q_norm.variance_epsilon,
|
||||
q_out,
|
||||
k_cache,
|
||||
v_cache,
|
||||
slot_mapping,
|
||||
forward_batch.out_cache_loc,
|
||||
self.num_heads,
|
||||
self.num_kv_heads,
|
||||
self.head_dim,
|
||||
self._fused_k_scale,
|
||||
self._fused_v_scale,
|
||||
None,
|
||||
None,
|
||||
False,
|
||||
False,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
|
||||
q = q_out.reshape(num_tokens, -1)
|
||||
return q, None, None
|
||||
|
||||
def forward(
|
||||
@@ -275,15 +295,23 @@ class Qwen3Attention(nn.Module):
|
||||
hidden_states = hidden_states.bfloat16()
|
||||
|
||||
save_kv_cache = True
|
||||
use_aiter_fused = (
|
||||
use_aiter_fused = False
|
||||
can_try_aiter_fused = (
|
||||
self.use_fused_qk_norm_mrope
|
||||
and forward_batch.forward_mode.is_decode()
|
||||
and get_exec().deterministic.rl_on_policy_target is None
|
||||
)
|
||||
if can_try_aiter_fused:
|
||||
k_cache, v_cache = get_token_to_kv_pool().get_kv_buffer(self.attn.layer_id)
|
||||
# The fused write targets the NHD layout in the activation dtype and
|
||||
# does not apply per-layer KV scales.
|
||||
use_aiter_fused = (
|
||||
k_cache.dim() == 3 and k_cache.dtype == hidden_states.dtype
|
||||
)
|
||||
|
||||
if use_aiter_fused:
|
||||
q, k, v = self.forward_prepare_aiter_fused_mrope(
|
||||
positions, hidden_states, forward_batch
|
||||
positions, hidden_states, forward_batch, k_cache, v_cache
|
||||
)
|
||||
save_kv_cache = False
|
||||
elif not _is_npu:
|
||||
|
||||
Reference in New Issue
Block a user